Date: Sun, 18 Oct 2015 20:20:21 +0000 (UTC) From: "Conrad E. Meyer" <cem@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r289542 - head/sys/dev/ntb/ntb_hw Message-ID: <201510182020.t9IKKLdU092945@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: cem Date: Sun Oct 18 20:20:20 2015 New Revision: 289542 URL: https://svnweb.freebsd.org/changeset/base/289542 Log: NTB: "Split ntb_hw_intel and ntb_transport drivers" This Linux commit was more or less a rewrite. Unfortunately, the commit log does not give a lot of context for the rewrite. I have tried to faithfully follow the changes made upstream, including matching function names where possible, while churning the FreeBSD driver as little as possible. This is the bulk of the rewrite. There are two groups of changes to follow in separate commits: fleshing out the rest of the changes to xeon_setup_b2b_mw(), and some changes to if_ntb. Yes, this is a big patch (3 files changed, 416 insertions(+), 237 deletions(-)), but the Linux patch was 13 files changed, 2,589 additions(+) and 2,195 deletions(-). Original Linux commit log: Change ntb_hw_intel to use the new NTB hardware abstraction layer. Split ntb_transport into its own driver. Change it to use the new NTB hardware abstraction layer. Authored by: Allen Hubbe Obtained from: Linux (e26a5843) (Dual BSD/GPL driver) Sponsored by: EMC / Isilon Storage Division Modified: head/sys/dev/ntb/ntb_hw/ntb_hw.c head/sys/dev/ntb/ntb_hw/ntb_hw.h head/sys/dev/ntb/ntb_hw/ntb_regs.h Modified: head/sys/dev/ntb/ntb_hw/ntb_hw.c ============================================================================== --- head/sys/dev/ntb/ntb_hw/ntb_hw.c Sun Oct 18 20:20:11 2015 (r289541) +++ head/sys/dev/ntb/ntb_hw/ntb_hw.c Sun Oct 18 20:20:20 2015 (r289542) @@ -1,5 +1,6 @@ /*- * Copyright (C) 2013 Intel Corporation + * Copyright (C) 2015 EMC Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -115,6 +116,33 @@ struct ntb_db_cb { bool reserved; }; +struct ntb_reg { + uint32_t ntb_ctl; + uint32_t lnk_sta; + uint8_t db_size; + unsigned mw_bar[NTB_MAX_BARS]; +}; + +struct ntb_alt_reg { + uint32_t db_bell; + uint32_t db_mask; + uint32_t spad; +}; + +struct ntb_xlat_reg { + uint64_t bar0_base; + uint64_t bar2_xlat; + uint64_t bar2_limit; +}; + +struct ntb_b2b_addr { + uint64_t bar0_addr; + uint64_t bar2_addr64; + uint64_t bar4_addr64; + uint64_t bar4_addr32; + uint64_t bar5_addr32; +}; + struct ntb_softc { device_t device; enum ntb_device_type type; @@ -135,14 +163,9 @@ struct ntb_softc { struct { uint32_t ldb; uint32_t ldb_mask; - uint32_t rdb; - uint32_t bar2_xlat; uint32_t bar4_xlat; uint32_t bar5_xlat; - uint32_t spad_remote; uint32_t spad_local; - uint32_t lnk_cntl; - uint32_t lnk_stat; uint32_t spci_cmd; } reg_ofs; uint32_t ppd; @@ -152,11 +175,36 @@ struct ntb_softc { uint8_t link_width; uint8_t link_speed; + /* Offset of peer bar0 in B2B BAR */ + uint64_t b2b_off; + /* Memory window used to access peer bar0 */ + uint8_t b2b_mw_idx; + uint8_t mw_count; uint8_t spad_count; uint8_t db_count; uint8_t db_vec_count; uint8_t db_vec_shift; + + /* Protects local DB mask and (h). */ +#define HW_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock) +#define HW_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock) +#define HW_ASSERT(sc,f) mtx_assert(&(sc)->db_mask_lock, (f)) + struct mtx db_mask_lock; + + uint32_t ntb_ctl; /* (h) - SOC only */ + uint32_t lnk_sta; /* (h) - SOC only */ + + uint64_t db_valid_mask; + uint64_t db_link_mask; + uint64_t db_mask; /* (h) */ + + int last_ts; /* ticks @ last irq */ + + const struct ntb_reg *reg; + const struct ntb_alt_reg *self_reg; + const struct ntb_alt_reg *peer_reg; + const struct ntb_xlat_reg *xlat_reg; }; #ifdef __i386__ @@ -189,9 +237,9 @@ bus_space_write_8(bus_space_tag_t tag, b #define ntb_reg_write(SIZE, offset, val) \ ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) #define ntb_mw_read(SIZE, offset) \ - ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->mw_count), offset) + ntb_bar_read(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), offset) #define ntb_mw_write(SIZE, offset, val) \ - ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->mw_count), \ + ntb_bar_write(SIZE, ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ offset, val) static int ntb_probe(device_t device); @@ -217,23 +265,27 @@ static inline uint64_t ntb_db_read(struc static inline void ntb_db_write(struct ntb_softc *, uint64_t regoff, uint64_t val); static inline void mask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx); static inline void unmask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx); +static inline void ntb_db_set_mask(struct ntb_softc *, uint64_t bits); +static inline void ntb_db_clear_mask(struct ntb_softc *, uint64_t bits); static int ntb_create_callbacks(struct ntb_softc *ntb, uint32_t num_vectors); static void ntb_free_callbacks(struct ntb_softc *ntb); static struct ntb_hw_info *ntb_get_device_info(uint32_t device_id); static void ntb_detect_max_mw(struct ntb_softc *ntb); static int ntb_detect_xeon(struct ntb_softc *ntb); static int ntb_detect_soc(struct ntb_softc *ntb); -static int ntb_setup_xeon(struct ntb_softc *ntb); -static int ntb_setup_soc(struct ntb_softc *ntb); +static int ntb_xeon_init_dev(struct ntb_softc *ntb); +static int ntb_soc_init_dev(struct ntb_softc *ntb); static void ntb_teardown_xeon(struct ntb_softc *ntb); static void configure_soc_secondary_side_bars(struct ntb_softc *ntb); static void configure_xeon_secondary_side_bars(struct ntb_softc *ntb); -static void ntb_handle_heartbeat(void *arg); +static int xeon_setup_b2b_mw(struct ntb_softc *, + const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr); +static void soc_link_hb(void *arg); static void ntb_handle_link_event(struct ntb_softc *ntb, int link_state); -static void ntb_hw_link_down(struct ntb_softc *ntb); -static void ntb_hw_link_up(struct ntb_softc *ntb); +static void ntb_link_disable(struct ntb_softc *ntb); +static void ntb_link_enable(struct ntb_softc *ntb); static void recover_soc_link(void *arg); -static int ntb_check_link_status(struct ntb_softc *ntb); +static int ntb_poll_link(struct ntb_softc *ntb); static void save_bar_parameters(struct ntb_pci_bar_info *bar); static struct ntb_hw_info pci_ids[] = { @@ -257,6 +309,61 @@ static struct ntb_hw_info pci_ids[] = { { 0x00000000, NULL, NTB_SOC, 0 } }; +static const struct ntb_reg soc_reg = { + .ntb_ctl = SOC_NTBCNTL_OFFSET, + .lnk_sta = SOC_LINK_STATUS_OFFSET, + .db_size = sizeof(uint64_t), + .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, +}; + +static const struct ntb_alt_reg soc_b2b_reg = { + .db_bell = SOC_B2B_DOORBELL_OFFSET, + .spad = SOC_B2B_SPAD_OFFSET, +}; + +static const struct ntb_xlat_reg soc_sec_xlat = { +#if 0 + /* "FIXME" says the Linux driver. */ + .bar0_base = SOC_SBAR0BASE_OFFSET, + .bar2_limit = SOC_SBAR2LMT_OFFSET, +#endif + .bar2_xlat = SOC_SBAR2XLAT_OFFSET, +}; + +static const struct ntb_reg xeon_reg = { + .ntb_ctl = XEON_NTBCNTL_OFFSET, + .lnk_sta = XEON_LINK_STATUS_OFFSET, + .db_size = sizeof(uint16_t), + .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 }, +}; + +static const struct ntb_alt_reg xeon_b2b_reg = { + .db_bell = XEON_B2B_DOORBELL_OFFSET, + .spad = XEON_B2B_SPAD_OFFSET, +}; + +static const struct ntb_xlat_reg xeon_sec_xlat = { + .bar0_base = XEON_SBAR0BASE_OFFSET, + .bar2_limit = XEON_SBAR2LMT_OFFSET, + .bar2_xlat = XEON_SBAR2XLAT_OFFSET, +}; + +static const struct ntb_b2b_addr xeon_b2b_usd_addr = { + .bar0_addr = XEON_B2B_BAR0_USD_ADDR, + .bar2_addr64 = XEON_B2B_BAR2_USD_ADDR64, + .bar4_addr64 = XEON_B2B_BAR4_USD_ADDR64, + .bar4_addr32 = XEON_B2B_BAR4_USD_ADDR32, + .bar5_addr32 = XEON_B2B_BAR5_USD_ADDR32, +}; + +static const struct ntb_b2b_addr xeon_b2b_dsd_addr = { + .bar0_addr = XEON_B2B_BAR0_DSD_ADDR, + .bar2_addr64 = XEON_B2B_BAR2_DSD_ADDR64, + .bar4_addr64 = XEON_B2B_BAR4_DSD_ADDR64, + .bar4_addr32 = XEON_B2B_BAR4_DSD_ADDR32, + .bar5_addr32 = XEON_B2B_BAR5_DSD_ADDR32, +}; + /* * OS <-> Driver interface structures */ @@ -311,10 +418,12 @@ ntb_attach(device_t device) ntb->device = device; ntb->type = p->type; ntb->features = p->features; + ntb->b2b_mw_idx = UINT8_MAX; /* Heartbeat timer for NTB_SOC since there is no link interrupt */ callout_init(&ntb->heartbeat_timer, 1); callout_init(&ntb->lr_timer, 1); + mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN); if (ntb->type == NTB_SOC) error = ntb_detect_soc(ntb); @@ -329,9 +438,9 @@ ntb_attach(device_t device) if (error) goto out; if (ntb->type == NTB_SOC) - error = ntb_setup_soc(ntb); + error = ntb_soc_init_dev(ntb); else - error = ntb_setup_xeon(ntb); + error = ntb_xeon_init_dev(ntb); if (error) goto out; error = ntb_init_isr(ntb); @@ -352,12 +461,16 @@ ntb_detach(device_t device) struct ntb_softc *ntb; ntb = DEVICE2SOFTC(device); + + ntb_db_set_mask(ntb, ntb->db_valid_mask); callout_drain(&ntb->heartbeat_timer); callout_drain(&ntb->lr_timer); if (ntb->type == NTB_XEON) ntb_teardown_xeon(ntb); ntb_teardown_interrupts(ntb); + mtx_destroy(&ntb->db_mask_lock); + /* * Redetect total MWs so we unmap properly -- in case we lowered the * maximum to work around Xeon errata. @@ -368,14 +481,17 @@ ntb_detach(device_t device) return (0); } +/* + * Driver internal routines + */ static inline enum ntb_bar ntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) { - KASSERT(mw < ntb->mw_count, ("%s: mw:%u > count:%u", __func__, mw, - (unsigned)ntb->mw_count)); + KASSERT(mw < ntb->mw_count || (mw != UINT8_MAX && mw == ntb->b2b_mw_idx), + ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count)); - return (NTB_B2B_BAR_1 + mw); + return (ntb->reg->mw_bar[mw]); } static int @@ -593,15 +709,16 @@ ntb_init_isr(struct ntb_softc *ntb) int rc; ntb->allocated_interrupts = 0; + ntb->last_ts = ticks; /* * On SOC, disable all interrupts. On XEON, disable all but Link * Interrupt. The rest will be unmasked as callbacks are registered. */ - mask = 0; + mask = ntb->db_valid_mask; if (ntb->type == NTB_XEON) - mask = (1 << XEON_DB_LINK); - ntb_db_write(ntb, ntb->reg_ofs.ldb_mask, ~mask); + mask &= ~ntb->db_link_mask; + ntb_db_set_mask(ntb, mask); num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), ntb->db_count); @@ -724,6 +841,14 @@ static inline void ntb_db_write(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) { + KASSERT((val & ~ntb->db_valid_mask) == 0, + ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, + (uintmax_t)(val & ~ntb->db_valid_mask), + (uintmax_t)ntb->db_valid_mask)); + + if (regoff == ntb->reg_ofs.ldb_mask) + HW_ASSERT(ntb, MA_OWNED); + if (ntb->type == NTB_SOC) { ntb_reg_write(8, regoff, val); return; @@ -733,14 +858,38 @@ ntb_db_write(struct ntb_softc *ntb, uint ntb_reg_write(2, regoff, (uint16_t)val); } -static void +static inline void +ntb_db_set_mask(struct ntb_softc *ntb, uint64_t bits) +{ + + HW_LOCK(ntb); + ntb->db_mask |= bits; + ntb_db_write(ntb, ntb->reg_ofs.ldb_mask, ntb->db_mask); + HW_UNLOCK(ntb); +} + +static inline void +ntb_db_clear_mask(struct ntb_softc *ntb, uint64_t bits) +{ + + KASSERT((bits & ~ntb->db_valid_mask) == 0, + ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, + (uintmax_t)(bits & ~ntb->db_valid_mask), + (uintmax_t)ntb->db_valid_mask)); + + HW_LOCK(ntb); + ntb->db_mask &= ~bits; + ntb_db_write(ntb, ntb->reg_ofs.ldb_mask, ntb->db_mask); + HW_UNLOCK(ntb); +} + +static inline void mask_ldb_interrupt(struct ntb_softc *ntb, unsigned int idx) { uint64_t mask; - mask = ntb_db_read(ntb, ntb->reg_ofs.ldb_mask); - mask |= 1 << (idx * ntb->db_vec_shift); - ntb_db_write(ntb, ntb->reg_ofs.ldb_mask, mask); + mask = 1ull << (idx * ntb->db_vec_shift); + ntb_db_set_mask(ntb, mask); } static inline void @@ -748,9 +897,8 @@ unmask_ldb_interrupt(struct ntb_softc *n { uint64_t mask; - mask = ntb_db_read(ntb, ntb->reg_ofs.ldb_mask); - mask &= ~(1 << (idx * ntb->db_vec_shift)); - ntb_db_write(ntb, ntb->reg_ofs.ldb_mask, mask); + mask = 1ull << (idx * ntb->db_vec_shift); + ntb_db_clear_mask(ntb, mask); } static inline uint64_t @@ -771,10 +919,11 @@ handle_irq(void *arg) uint64_t vec_mask; int rc; + ntb->last_ts = ticks; vec_mask = ntb_vec_mask(ntb, db_cb->db_num); - if (ntb->type == NTB_XEON && (vec_mask & XEON_DB_LINK_BIT) != 0) { - rc = ntb_check_link_status(ntb); + if ((vec_mask & ntb->db_link_mask) != 0) { + rc = ntb_poll_link(ntb); if (rc != 0) device_printf(ntb->device, "Error determining link status\n"); @@ -855,7 +1004,7 @@ static void ntb_teardown_xeon(struct ntb_softc *ntb) { - ntb_hw_link_down(ntb); + ntb_link_disable(ntb); } static void @@ -889,6 +1038,10 @@ ntb_detect_xeon(struct ntb_softc *ntb) if ((ppd & XEON_PPD_SPLIT_BAR) != 0) ntb->features |= NTB_SPLIT_BAR; + /* SB01BASE_LOCKUP errata is a superset of SDOORBELL errata */ + if (HAS_FEATURE(NTB_SB01BASE_LOCKUP)) + ntb->features |= NTB_SDOORBELL_LOCKUP; + conn_type = ppd & XEON_PPD_CONN_TYPE; switch (conn_type) { case NTB_CONN_B2B: @@ -930,48 +1083,34 @@ ntb_detect_soc(struct ntb_softc *ntb) } static int -ntb_setup_xeon(struct ntb_softc *ntb) +ntb_xeon_init_dev(struct ntb_softc *ntb) { + int rc; ntb->reg_ofs.ldb = XEON_PDOORBELL_OFFSET; ntb->reg_ofs.ldb_mask = XEON_PDBMSK_OFFSET; ntb->reg_ofs.spad_local = XEON_SPAD_OFFSET; - ntb->reg_ofs.bar2_xlat = XEON_SBAR2XLAT_OFFSET; ntb->reg_ofs.bar4_xlat = XEON_SBAR4XLAT_OFFSET; if (HAS_FEATURE(NTB_SPLIT_BAR)) ntb->reg_ofs.bar5_xlat = XEON_SBAR5XLAT_OFFSET; + ntb->reg_ofs.spci_cmd = XEON_PCICMD_OFFSET; - switch (ntb->conn_type) { - case NTB_CONN_B2B: - /* - * reg_ofs.rdb and reg_ofs.spad_remote are effectively ignored - * with the NTB_SDOORBELL_LOCKUP errata mode enabled. (See - * ntb_ring_doorbell() and ntb_read/write_remote_spad().) - */ - ntb->reg_ofs.rdb = XEON_B2B_DOORBELL_OFFSET; - ntb->reg_ofs.spad_remote = XEON_B2B_SPAD_OFFSET; - - ntb->spad_count = XEON_SPAD_COUNT; - break; - - case NTB_CONN_RP: - /* - * Every Xeon today needs NTB_SDOORBELL_LOCKUP, so punt on RP for - * now. - */ - KASSERT(HAS_FEATURE(NTB_SDOORBELL_LOCKUP), - ("Xeon without MW errata unimplemented")); - device_printf(ntb->device, - "NTB-RP disabled to due hardware errata.\n"); - return (ENXIO); + ntb->spad_count = XEON_SPAD_COUNT; + ntb->db_count = XEON_DB_COUNT; + ntb->db_link_mask = XEON_DB_LINK_BIT; + ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT; + ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT; - case NTB_CONN_TRANSPARENT: - default: + if (ntb->conn_type != NTB_CONN_B2B) { device_printf(ntb->device, "Connection type %d not supported\n", ntb->conn_type); return (ENXIO); } + ntb->reg = &xeon_reg; + ntb->peer_reg = &xeon_b2b_reg; + ntb->xlat_reg = &xeon_sec_xlat; + /* * There is a Xeon hardware errata related to writes to SDOORBELL or * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, @@ -985,17 +1124,21 @@ ntb_setup_xeon(struct ntb_softc *ntb) * write the limit registers first just in case. */ if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) { + /* Reserve the last MW for mapping remote spad */ + ntb->b2b_mw_idx = ntb->mw_count - 1; + ntb->mw_count--; /* * Set the Limit register to 4k, the minimum size, to prevent * an illegal access. - * - * XXX: Should this be PBAR5LMT / get_mw_size(, max_mw - 1)? */ - ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, - ntb_get_mw_size(ntb, 1) + 0x1000); - /* Reserve the last MW for mapping remote spad */ - ntb->mw_count--; - } else + if (HAS_FEATURE(NTB_SPLIT_BAR)) { + ntb_reg_write(4, XEON_PBAR4LMT_OFFSET, 0); + ntb_reg_write(4, XEON_PBAR5LMT_OFFSET, + ntb_get_mw_size(ntb, ntb->b2b_mw_idx) + 0x1000); + } else + ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, + ntb_get_mw_size(ntb, ntb->b2b_mw_idx) + 0x1000); + } else { /* * Disable the limit register, just in case it is set to * something silly. A 64-bit write will also clear PBAR5LMT in @@ -1003,66 +1146,61 @@ ntb_setup_xeon(struct ntb_softc *ntb) */ ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); - ntb->reg_ofs.lnk_cntl = XEON_NTBCNTL_OFFSET; - ntb->reg_ofs.lnk_stat = XEON_LINK_STATUS_OFFSET; - ntb->reg_ofs.spci_cmd = XEON_PCICMD_OFFSET; - - ntb->db_count = XEON_DB_COUNT; - ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT; - ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT; + /* + * HW Errata on bit 14 of b2bdoorbell register. Writes will not be + * mirrored to the remote system. Shrink the number of bits by one, + * since bit 14 is the last bit. + * + * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register + * anyway. Nor for non-B2B connection types. + */ + if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14)) + ntb->db_count = XEON_DB_COUNT - 1; + } - /* - * HW Errata on bit 14 of b2bdoorbell register. Writes will not be - * mirrored to the remote system. Shrink the number of bits by one, - * since bit 14 is the last bit. - * - * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register - * anyway. Nor for non-B2B connection types. - */ - if (HAS_FEATURE(NTB_B2BDOORBELL_BIT14) && - !HAS_FEATURE(NTB_SDOORBELL_LOCKUP) && - ntb->conn_type == NTB_CONN_B2B) - ntb->db_count = XEON_DB_COUNT - 1; + ntb->db_valid_mask = (1ull << ntb->db_count) - 1; - configure_xeon_secondary_side_bars(ntb); + if (ntb->dev_type == NTB_DEV_USD) + rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr, + &xeon_b2b_usd_addr); + else + rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr, + &xeon_b2b_dsd_addr); + if (rc != 0) + return (rc); /* Enable Bus Master and Memory Space on the secondary side */ - if (ntb->conn_type == NTB_CONN_B2B) - ntb_reg_write(2, ntb->reg_ofs.spci_cmd, - PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); + ntb_reg_write(2, ntb->reg_ofs.spci_cmd, + PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); /* Enable link training */ - ntb_hw_link_up(ntb); + ntb_link_enable(ntb); return (0); } static int -ntb_setup_soc(struct ntb_softc *ntb) +ntb_soc_init_dev(struct ntb_softc *ntb) { KASSERT(ntb->conn_type == NTB_CONN_B2B, ("Unsupported NTB configuration (%d)\n", ntb->conn_type)); - /* Initiate PCI-E link training */ - pci_write_config(ntb->device, NTB_PPD_OFFSET, - ntb->ppd | SOC_PPD_INIT_LINK, 4); - ntb->reg_ofs.ldb = SOC_PDOORBELL_OFFSET; ntb->reg_ofs.ldb_mask = SOC_PDBMSK_OFFSET; - ntb->reg_ofs.rdb = SOC_B2B_DOORBELL_OFFSET; - ntb->reg_ofs.bar2_xlat = SOC_SBAR2XLAT_OFFSET; ntb->reg_ofs.bar4_xlat = SOC_SBAR4XLAT_OFFSET; - ntb->reg_ofs.lnk_cntl = SOC_NTBCNTL_OFFSET; - ntb->reg_ofs.lnk_stat = SOC_LINK_STATUS_OFFSET; ntb->reg_ofs.spad_local = SOC_SPAD_OFFSET; - ntb->reg_ofs.spad_remote = SOC_B2B_SPAD_OFFSET; ntb->reg_ofs.spci_cmd = SOC_PCICMD_OFFSET; - ntb->spad_count = SOC_SPAD_COUNT; + ntb->spad_count = SOC_SPAD_COUNT; ntb->db_count = SOC_DB_COUNT; ntb->db_vec_count = SOC_DB_MSIX_VECTOR_COUNT; ntb->db_vec_shift = SOC_DB_MSIX_VECTOR_SHIFT; + ntb->db_valid_mask = (1ull << ntb->db_count) - 1; + + ntb->reg = &soc_reg; + ntb->peer_reg = &soc_b2b_reg; + ntb->xlat_reg = &soc_sec_xlat; /* * FIXME - MSI-X bug on early SOC HW, remove once internal issue is @@ -1076,118 +1214,95 @@ ntb_setup_soc(struct ntb_softc *ntb) ntb_reg_write(2, ntb->reg_ofs.spci_cmd, PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); - callout_reset(&ntb->heartbeat_timer, 0, ntb_handle_heartbeat, ntb); + /* Initiate PCI-E link training */ + ntb_link_enable(ntb); + + callout_reset(&ntb->heartbeat_timer, 0, soc_link_hb, ntb); return (0); } +/* XXX: Linux driver doesn't seem to do any of this for SoC. */ static void configure_soc_secondary_side_bars(struct ntb_softc *ntb) { if (ntb->dev_type == NTB_DEV_USD) { ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, - XEON_B2B_BAR2_DSD_ADDR); - ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, XEON_B2B_BAR4_DSD_ADDR); - ntb_reg_write(8, SOC_MBAR23_OFFSET, XEON_B2B_BAR2_USD_ADDR); - ntb_reg_write(8, SOC_MBAR45_OFFSET, XEON_B2B_BAR4_USD_ADDR); + XEON_B2B_BAR2_DSD_ADDR64); + ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, + XEON_B2B_BAR4_DSD_ADDR64); + ntb_reg_write(8, SOC_MBAR23_OFFSET, XEON_B2B_BAR2_USD_ADDR64); + ntb_reg_write(8, SOC_MBAR45_OFFSET, XEON_B2B_BAR4_USD_ADDR64); } else { ntb_reg_write(8, SOC_PBAR2XLAT_OFFSET, - XEON_B2B_BAR2_USD_ADDR); - ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, XEON_B2B_BAR4_USD_ADDR); - ntb_reg_write(8, SOC_MBAR23_OFFSET, XEON_B2B_BAR2_DSD_ADDR); - ntb_reg_write(8, SOC_MBAR45_OFFSET, XEON_B2B_BAR4_DSD_ADDR); + XEON_B2B_BAR2_USD_ADDR64); + ntb_reg_write(8, SOC_PBAR4XLAT_OFFSET, + XEON_B2B_BAR4_USD_ADDR64); + ntb_reg_write(8, SOC_MBAR23_OFFSET, XEON_B2B_BAR2_DSD_ADDR64); + ntb_reg_write(8, SOC_MBAR45_OFFSET, XEON_B2B_BAR4_DSD_ADDR64); } } -static void -configure_xeon_secondary_side_bars(struct ntb_softc *ntb) +static int +xeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, + const struct ntb_b2b_addr *peer_addr) { - if (ntb->dev_type == NTB_DEV_USD) { - ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, - XEON_B2B_BAR2_DSD_ADDR); - if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) - ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, - XEON_B2B_BAR0_DSD_ADDR); - else { - if (HAS_FEATURE(NTB_SPLIT_BAR)) { - ntb_reg_write(4, XEON_PBAR4XLAT_OFFSET, - XEON_B2B_BAR4_DSD_ADDR); - ntb_reg_write(4, XEON_PBAR5XLAT_OFFSET, - XEON_B2B_BAR5_DSD_ADDR); - } else - ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, - XEON_B2B_BAR4_DSD_ADDR); - /* - * B2B_XLAT_OFFSET is a 64-bit register but can only be - * written 32 bits at a time. - */ - ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, - XEON_B2B_BAR0_DSD_ADDR & 0xffffffff); - ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, - XEON_B2B_BAR0_DSD_ADDR >> 32); - } - ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, - XEON_B2B_BAR0_USD_ADDR); - ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, - XEON_B2B_BAR2_USD_ADDR); + /* Local addresses */ + ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, addr->bar2_addr64); + if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) + ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, addr->bar0_addr); + else { if (HAS_FEATURE(NTB_SPLIT_BAR)) { - ntb_reg_write(4, XEON_SBAR4BASE_OFFSET, - XEON_B2B_BAR4_USD_ADDR); - ntb_reg_write(4, XEON_SBAR5BASE_OFFSET, - XEON_B2B_BAR5_USD_ADDR); + ntb_reg_write(4, XEON_PBAR4XLAT_OFFSET, + addr->bar4_addr32); + ntb_reg_write(4, XEON_PBAR5XLAT_OFFSET, + addr->bar5_addr32); } else - ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, - XEON_B2B_BAR4_USD_ADDR); - } else { - ntb_reg_write(8, XEON_PBAR2XLAT_OFFSET, - XEON_B2B_BAR2_USD_ADDR); - if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, - XEON_B2B_BAR0_USD_ADDR); - else { - if (HAS_FEATURE(NTB_SPLIT_BAR)) { - ntb_reg_write(4, XEON_PBAR4XLAT_OFFSET, - XEON_B2B_BAR4_USD_ADDR); - ntb_reg_write(4, XEON_PBAR5XLAT_OFFSET, - XEON_B2B_BAR5_USD_ADDR); - } else - ntb_reg_write(8, XEON_PBAR4XLAT_OFFSET, - XEON_B2B_BAR4_USD_ADDR); - /* - * B2B_XLAT_OFFSET is a 64-bit register but can only be - * written 32 bits at a time. - */ - ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, - XEON_B2B_BAR0_USD_ADDR & 0xffffffff); - ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, - XEON_B2B_BAR0_USD_ADDR >> 32); - } - ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, - XEON_B2B_BAR0_DSD_ADDR); - ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, - XEON_B2B_BAR2_DSD_ADDR); - if (HAS_FEATURE(NTB_SPLIT_BAR)) { - ntb_reg_write(4, XEON_SBAR4BASE_OFFSET, - XEON_B2B_BAR4_DSD_ADDR); - ntb_reg_write(4, XEON_SBAR5BASE_OFFSET, - XEON_B2B_BAR5_DSD_ADDR); - } else - ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, - XEON_B2B_BAR4_DSD_ADDR); + addr->bar4_addr64); + /* + * B2B_XLAT_OFFSET is a 64-bit register but can only be + * written 32 bits at a time. + */ + ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, + addr->bar0_addr & 0xffffffff); + ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, addr->bar0_addr >> 32); } + + /* Peer addresses */ + ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, peer_addr->bar0_addr); + ntb_reg_write(8, XEON_SBAR2BASE_OFFSET, peer_addr->bar2_addr64); + if (HAS_FEATURE(NTB_SPLIT_BAR)) { + ntb_reg_write(4, XEON_SBAR4BASE_OFFSET, + peer_addr->bar4_addr32); + ntb_reg_write(4, XEON_SBAR5BASE_OFFSET, + peer_addr->bar5_addr32); + } else + ntb_reg_write(8, XEON_SBAR4BASE_OFFSET, + peer_addr->bar4_addr64); + return (0); } /* SOC does not have link status interrupt, poll on that platform */ static void -ntb_handle_heartbeat(void *arg) +soc_link_hb(void *arg) { struct ntb_softc *ntb = arg; uint32_t status32; int rc; - rc = ntb_check_link_status(ntb); + /* + * Delay polling the link status if an interrupt was received, unless + * the cached link status says the link is down. + */ + if ((long)ticks - ((long)ntb->last_ts + NTB_HB_TIMEOUT * hz) < 0 && + (ntb->ntb_ctl & SOC_CNTL_LINK_DOWN) == 0) + goto out; + + + rc = ntb_poll_link(ntb); if (rc != 0) device_printf(ntb->device, "Error determining link status\n"); @@ -1202,8 +1317,9 @@ ntb_handle_heartbeat(void *arg) } } - callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, - ntb_handle_heartbeat, ntb); +out: + callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, soc_link_hb, + ntb); } static void @@ -1261,7 +1377,7 @@ ntb_handle_link_event(struct ntb_softc * if (ntb->type == NTB_SOC || ntb->conn_type == NTB_CONN_TRANSPARENT) - status = ntb_reg_read(2, ntb->reg_ofs.lnk_stat); + status = ntb_reg_read(2, ntb->reg->lnk_sta); else status = pci_read_config(ntb->device, XEON_LINK_STATUS_OFFSET, 2); @@ -1270,7 +1386,7 @@ ntb_handle_link_event(struct ntb_softc * device_printf(ntb->device, "Link Width %d, Link Speed %d\n", ntb->link_width, ntb->link_speed); callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, - ntb_handle_heartbeat, ntb); + soc_link_hb, ntb); } else { device_printf(ntb->device, "Link Down\n"); ntb->link_status = NTB_LINK_DOWN; @@ -1284,26 +1400,32 @@ ntb_handle_link_event(struct ntb_softc * } static void -ntb_hw_link_up(struct ntb_softc *ntb) +ntb_link_enable(struct ntb_softc *ntb) { uint32_t cntl; + if (ntb->type == NTB_SOC) { + pci_write_config(ntb->device, NTB_PPD_OFFSET, + ntb->ppd | SOC_PPD_INIT_LINK, 4); + return; + } + if (ntb->conn_type == NTB_CONN_TRANSPARENT) { ntb_handle_link_event(ntb, NTB_LINK_UP); return; } - cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); + cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; if (HAS_FEATURE(NTB_SPLIT_BAR)) cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP; - ntb_reg_write(4, ntb->reg_ofs.lnk_cntl, cntl); + ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); } static void -ntb_hw_link_down(struct ntb_softc *ntb) +ntb_link_disable(struct ntb_softc *ntb) { uint32_t cntl; @@ -1312,13 +1434,13 @@ ntb_hw_link_down(struct ntb_softc *ntb) return; } - cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); + cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); if (HAS_FEATURE(NTB_SPLIT_BAR)) cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP); cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; - ntb_reg_write(4, ntb->reg_ofs.lnk_cntl, cntl); + ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); } static void @@ -1327,7 +1449,6 @@ recover_soc_link(void *arg) struct ntb_softc *ntb = arg; uint8_t speed, width; uint32_t status32; - uint16_t status16; soc_perform_link_restart(ntb); @@ -1348,19 +1469,19 @@ recover_soc_link(void *arg) if ((status32 & SOC_IBIST_ERR_OFLOW) != 0) goto retry; - status32 = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); + status32 = ntb_reg_read(4, ntb->reg->ntb_ctl); if ((status32 & SOC_CNTL_LINK_DOWN) != 0) goto out; - status16 = ntb_reg_read(2, ntb->reg_ofs.lnk_stat); - width = (status16 & NTB_LINK_WIDTH_MASK) >> 4; - speed = (status16 & NTB_LINK_SPEED_MASK); + status32 = ntb_reg_read(4, ntb->reg->lnk_sta); + width = (status32 & NTB_LINK_WIDTH_MASK) >> 4; + speed = (status32 & NTB_LINK_SPEED_MASK); if (ntb->link_width != width || ntb->link_speed != speed) goto retry; out: - callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, - ntb_handle_heartbeat, ntb); + callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, soc_link_hb, + ntb); return; retry: @@ -1369,21 +1490,32 @@ retry: } static int -ntb_check_link_status(struct ntb_softc *ntb) +ntb_poll_link(struct ntb_softc *ntb) { int link_state; uint32_t ntb_cntl; uint16_t status; if (ntb->type == NTB_SOC) { - ntb_cntl = ntb_reg_read(4, ntb->reg_ofs.lnk_cntl); + HW_LOCK(ntb); + ntb_cntl = ntb_reg_read(4, ntb->reg->ntb_ctl); + if (ntb_cntl == ntb->ntb_ctl) { + HW_UNLOCK(ntb); + return (0); + } + ntb->ntb_ctl = ntb_cntl; + ntb->lnk_sta = ntb_reg_read(4, ntb->reg->lnk_sta); + HW_UNLOCK(ntb); + if ((ntb_cntl & SOC_CNTL_LINK_DOWN) != 0) link_state = NTB_LINK_DOWN; else link_state = NTB_LINK_UP; } else { - status = pci_read_config(ntb->device, XEON_LINK_STATUS_OFFSET, - 2); + status = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); + if (status == ntb->lnk_sta) + return (0); + ntb->lnk_sta = status; if ((status & NTB_LINK_STATUS_ACTIVE) != 0) link_state = NTB_LINK_UP; @@ -1392,10 +1524,32 @@ ntb_check_link_status(struct ntb_softc * } ntb_handle_link_event(ntb, link_state); - return (0); } +static void +ntb_irq_work(void *arg) +{ + struct ntb_db_cb *db_cb = arg; + struct ntb_softc *ntb; + int rc; + + rc = db_cb->callback(db_cb->data, db_cb->db_num); + /* Poll if forward progress was made. */ + if (rc != 0) { + callout_reset(&db_cb->irq_work, 0, ntb_irq_work, db_cb); + return; + } + + /* Unmask interrupt if no progress was made. */ + ntb = db_cb->ntb; + unmask_ldb_interrupt(ntb, db_cb->db_num); +} + +/* + * Public API to the rest of the OS + */ + /** * ntb_register_event_callback() - register event callback * @ntb: pointer to ntb_softc instance @@ -1431,25 +1585,6 @@ ntb_unregister_event_callback(struct ntb ntb->event_cb = NULL; } -static void -ntb_irq_work(void *arg) -{ - struct ntb_db_cb *db_cb = arg; - struct ntb_softc *ntb; - int rc; - - rc = db_cb->callback(db_cb->data, db_cb->db_num); - /* Poll if forward progress was made. */ - if (rc != 0) { - callout_reset(&db_cb->irq_work, 0, ntb_irq_work, db_cb); - return; - } - - /* Unmask interrupt if no progress was made. */ - ntb = db_cb->ntb; - unmask_ldb_interrupt(ntb, db_cb->db_num); -} - /** * ntb_register_db_callback() - register a callback for doorbell interrupt * @ntb: pointer to ntb_softc instance @@ -1663,7 +1798,7 @@ ntb_write_remote_spad(struct ntb_softc * if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) ntb_mw_write(4, XEON_SHADOW_SPAD_OFFSET + idx * 4, val); else - ntb_reg_write(4, ntb->reg_ofs.spad_remote + idx * 4, val); + ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val); return (0); } @@ -1689,7 +1824,7 @@ ntb_read_remote_spad(struct ntb_softc *n if (HAS_FEATURE(NTB_SDOORBELL_LOCKUP)) *val = ntb_mw_read(4, XEON_SHADOW_SPAD_OFFSET + idx * 4); else - *val = ntb_reg_read(4, ntb->reg_ofs.spad_remote + idx * 4); + *val = ntb_reg_read(4, ntb->peer_reg->spad + idx * 4); return (0); } @@ -1714,7 +1849,7 @@ ntb_get_mw_vbase(struct ntb_softc *ntb, return (ntb->bar_info[ntb_mw_to_bar(ntb, mw)].vbase); } -vm_paddr_t +bus_addr_t ntb_get_mw_pbase(struct ntb_softc *ntb, unsigned int mw) { @@ -1762,7 +1897,7 @@ ntb_set_mw_addr(struct ntb_softc *ntb, u switch (ntb_mw_to_bar(ntb, mw)) { *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201510182020.t9IKKLdU092945>