From owner-svn-src-all@FreeBSD.ORG Wed Jul 30 17:58:18 2014 Return-Path: Delivered-To: svn-src-all@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) (using TLSv1 with cipher ADH-AES256-SHA (256/256 bits)) (No client certificate requested) by hub.freebsd.org (Postfix) with ESMTPS id 6CC3EC17; Wed, 30 Jul 2014 17:58:18 +0000 (UTC) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 59D362053; Wed, 30 Jul 2014 17:58:18 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.9/8.14.9) with ESMTP id s6UHwISC048531; Wed, 30 Jul 2014 17:58:18 GMT (envelope-from smh@svn.freebsd.org) Received: (from smh@localhost) by svn.freebsd.org (8.14.9/8.14.9/Submit) id s6UHwHII048526; Wed, 30 Jul 2014 17:58:17 GMT (envelope-from smh@svn.freebsd.org) Message-Id: <201407301758.s6UHwHII048526@svn.freebsd.org> From: Steven Hartland Date: Wed, 30 Jul 2014 17:58:17 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r269314 - head/sys/dev/mps X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.18 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 30 Jul 2014 17:58:18 -0000 Author: smh Date: Wed Jul 30 17:58:17 2014 New Revision: 269314 URL: http://svnweb.freebsd.org/changeset/base/269314 Log: Bring in LSI's phase16 - phase18 changes * Implements Start Stop Unit for SATA direct-attach devices in IR mode to avoid data corruption. * Use CAM_DEV_NOT_THERE instead of CAM_SEL_TIMEOUT and CAM_TID_INVALID Obtained from: LSI MFC after: 2 weeks Modified: head/sys/dev/mps/mps.c head/sys/dev/mps/mps_sas.c head/sys/dev/mps/mps_sas.h head/sys/dev/mps/mps_sas_lsi.c head/sys/dev/mps/mpsvar.h Modified: head/sys/dev/mps/mps.c ============================================================================== --- head/sys/dev/mps/mps.c Wed Jul 30 17:57:36 2014 (r269313) +++ head/sys/dev/mps/mps.c Wed Jul 30 17:58:17 2014 (r269314) @@ -140,6 +140,7 @@ mps_diag_reset(struct mps_softc *sc,int { uint32_t reg; int i, error, tries = 0; + uint8_t first_wait_done = FALSE; mps_dprint(sc, MPS_TRACE, "%s\n", __func__); @@ -182,15 +183,32 @@ mps_diag_reset(struct mps_softc *sc,int /* Wait up to 300 seconds in 50ms intervals */ error = ETIMEDOUT; - for (i = 0; i < 60000; i++) { - /* wait 50 msec */ - if (mtx_owned(&sc->mps_mtx) && sleep_flag == CAN_SLEEP) - msleep(&sc->msleep_fake_chan, &sc->mps_mtx, 0, - "mpsdiag", hz/20); - else if (sleep_flag == CAN_SLEEP) - pause("mpsdiag", hz/20); - else - DELAY(50 * 1000); + for (i = 0; i < 6000; i++) { + /* + * Wait 50 msec. If this is the first time through, wait 256 + * msec to satisfy Diag Reset timing requirements. + */ + if (first_wait_done) { + if (mtx_owned(&sc->mps_mtx) && sleep_flag == CAN_SLEEP) + msleep(&sc->msleep_fake_chan, &sc->mps_mtx, 0, + "mpsdiag", hz/20); + else if (sleep_flag == CAN_SLEEP) + pause("mpsdiag", hz/20); + else + DELAY(50 * 1000); + } else { + DELAY(256 * 1000); + first_wait_done = TRUE; + } + /* + * Check for the RESET_ADAPTER bit to be cleared first, then + * wait for the RESET state to be cleared, which takes a little + * longer. + */ + reg = mps_regread(sc, MPI2_HOST_DIAGNOSTIC_OFFSET); + if (reg & MPI2_DIAG_RESET_ADAPTER) { + continue; + } reg = mps_regread(sc, MPI2_DOORBELL_OFFSET); if ((reg & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_RESET) { error = 0; @@ -236,7 +254,7 @@ mps_transition_ready(struct mps_softc *s sleep_flags = (sc->mps_flags & MPS_FLAGS_ATTACH_DONE) ? CAN_SLEEP:NO_SLEEP; error = 0; - while (tries++ < 5) { + while (tries++ < 1200) { reg = mps_regread(sc, MPI2_DOORBELL_OFFSET); mps_dprint(sc, MPS_INIT, "Doorbell= 0x%x\n", reg); @@ -656,6 +674,9 @@ int mps_reinit(struct mps_softc *sc) { int error; + struct mpssas_softc *sassc; + + sassc = sc->sassc; MPS_FUNCTRACE(sc); @@ -736,6 +757,8 @@ mps_reinit(struct mps_softc *sc) mps_dprint(sc, MPS_INFO, "%s finished sc %p post %u free %u\n", __func__, sc, sc->replypostindex, sc->replyfreeindex); + mpssas_release_simq_reinit(sassc); + return 0; } @@ -2510,6 +2533,7 @@ int mps_request_polled(struct mps_softc *sc, struct mps_command *cm) { int error, timeout = 0, rc; + struct timeval cur_time, start_time; error = 0; @@ -2517,22 +2541,33 @@ mps_request_polled(struct mps_softc *sc, cm->cm_complete = NULL; mps_map_command(sc, cm); + getmicrotime(&start_time); while ((cm->cm_flags & MPS_CM_FLAGS_COMPLETE) == 0) { mps_intr_locked(sc); - DELAY(50 * 1000); - if (timeout++ > 1000) { + if (mtx_owned(&sc->mps_mtx)) + msleep(&sc->msleep_fake_chan, &sc->mps_mtx, 0, + "mpspoll", hz/20); + else + pause("mpsdiag", hz/20); + + /* + * Check for real-time timeout and fail if more than 60 seconds. + */ + getmicrotime(&cur_time); + timeout = cur_time.tv_sec - start_time.tv_sec; + if (timeout > 60) { mps_dprint(sc, MPS_FAULT, "polling failed\n"); error = ETIMEDOUT; break; } } - + if (error) { mps_dprint(sc, MPS_FAULT, "Calling Reinit from %s\n", __func__); rc = mps_reinit(sc); - mps_dprint(sc, MPS_FAULT, "Reinit %s\n", - (rc == 0) ? "success" : "failed"); + mps_dprint(sc, MPS_FAULT, "Reinit %s\n", (rc == 0) ? "success" : + "failed"); } return (error); Modified: head/sys/dev/mps/mps_sas.c ============================================================================== --- head/sys/dev/mps/mps_sas.c Wed Jul 30 17:57:36 2014 (r269313) +++ head/sys/dev/mps/mps_sas.c Wed Jul 30 17:58:17 2014 (r269314) @@ -191,6 +191,16 @@ mpssas_startup_increment(struct mpssas_s } void +mpssas_release_simq_reinit(struct mpssas_softc *sassc) +{ + if (sassc->flags & MPSSAS_QUEUE_FROZEN) { + sassc->flags &= ~MPSSAS_QUEUE_FROZEN; + xpt_release_simq(sassc->sim, 1); + mps_dprint(sassc->sc, MPS_INFO, "Unfreezing SIM queue\n"); + } +} + +void mpssas_startup_decrement(struct mpssas_softc *sassc) { MPS_FUNCTRACE(sassc->sc); @@ -1003,7 +1013,7 @@ mpssas_action(struct cam_sim *sim, union cts->ccb_h.target_id)); targ = &sassc->targets[cts->ccb_h.target_id]; if (targ->handle == 0x0) { - mpssas_set_ccbstatus(ccb, CAM_SEL_TIMEOUT); + mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); break; } @@ -1120,6 +1130,14 @@ mpssas_complete_all_commands(struct mps_ wakeup(cm); completed = 1; } + + if (cm->cm_sc->io_cmds_active != 0) { + cm->cm_sc->io_cmds_active--; + } else { + mps_dprint(cm->cm_sc, MPS_INFO, "Warning: " + "io_cmds_active is out of sync - resynching to " + "0\n"); + } if ((completed == 0) && (cm->cm_state != MPS_CM_STATE_FREE)) { /* this should never happen, but if it does, log */ @@ -1647,14 +1665,14 @@ mpssas_action_scsiio(struct mpssas_softc if (targ->handle == 0x0) { mps_dprint(sc, MPS_ERROR, "%s NULL handle for target %u\n", __func__, csio->ccb_h.target_id); - mpssas_set_ccbstatus(ccb, CAM_SEL_TIMEOUT); + mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); xpt_done(ccb); return; } if (targ->flags & MPS_TARGET_FLAGS_RAID_COMPONENT) { mps_dprint(sc, MPS_ERROR, "%s Raid component no SCSI IO " "supported %u\n", __func__, csio->ccb_h.target_id); - mpssas_set_ccbstatus(ccb, CAM_TID_INVALID); + mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); xpt_done(ccb); return; } @@ -1685,13 +1703,16 @@ mpssas_action_scsiio(struct mpssas_softc if ((sc->mps_flags & MPS_FLAGS_SHUTDOWN) != 0) { mps_dprint(sc, MPS_INFO, "%s shutting down\n", __func__); - mpssas_set_ccbstatus(ccb, CAM_TID_INVALID); + mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); xpt_done(ccb); return; } cm = mps_alloc_command(sc); - if (cm == NULL) { + if (cm == NULL || (sc->mps_flags & MPS_FLAGS_DIAGRESET)) { + if (cm != NULL) { + mps_free_command(sc, cm); + } if ((sassc->flags & MPSSAS_QUEUE_FROZEN) == 0) { xpt_freeze_simq(sassc->sim, 1); sassc->flags |= MPSSAS_QUEUE_FROZEN; @@ -2170,6 +2191,18 @@ mpssas_scsiio_complete(struct mps_softc } } + /* + * If this is a Start Stop Unit command and it was issued by the driver + * during shutdown, decrement the refcount to account for all of the + * commands that were sent. All SSU commands should be completed before + * shutdown completes, meaning SSU_refcount will be 0 after SSU_started + * is TRUE. + */ + if (sc->SSU_started && (csio->cdb_io.cdb_bytes[0] == START_STOP_UNIT)) { + mps_dprint(sc, MPS_INFO, "Decrementing SSU count.\n"); + sc->SSU_refcount--; + } + /* Take the fast path to completion */ if (cm->cm_reply == NULL) { if (mpssas_get_ccbstatus(ccb) == CAM_REQ_INPROG) { @@ -2999,7 +3032,7 @@ mpssas_action_smpio(struct mpssas_softc mps_dprint(sc, MPS_ERROR, "%s: handle %d does not have a valid " "parent handle!\n", __func__, targ->handle); - mpssas_set_ccbstatus(ccb, CAM_REQ_INVALID); + mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); goto bailout; } #ifdef OLD_MPS_PROBE @@ -3010,7 +3043,7 @@ mpssas_action_smpio(struct mpssas_softc mps_dprint(sc, MPS_ERROR, "%s: handle %d does not have a valid " "parent target!\n", __func__, targ->handle); - mpssas_set_ccbstatus(ccb, CAM_REQ_INVALID); + mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); goto bailout; } @@ -3020,7 +3053,7 @@ mpssas_action_smpio(struct mpssas_softc "%s: handle %d parent %d does not " "have an SMP target!\n", __func__, targ->handle, parent_target->handle); - mpssas_set_ccbstatus(ccb, CAM_REQ_INVALID); + mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); goto bailout; } @@ -3033,7 +3066,7 @@ mpssas_action_smpio(struct mpssas_softc "%s: handle %d parent %d does not " "have an SMP target!\n", __func__, targ->handle, targ->parent_handle); - mpssas_set_ccbstatus(ccb, CAM_REQ_INVALID); + mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); goto bailout; } @@ -3042,7 +3075,7 @@ mpssas_action_smpio(struct mpssas_softc "%s: handle %d parent handle %d does " "not have a valid SAS address!\n", __func__, targ->handle, targ->parent_handle); - mpssas_set_ccbstatus(ccb, CAM_REQ_INVALID); + mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); goto bailout; } @@ -3055,7 +3088,7 @@ mpssas_action_smpio(struct mpssas_softc mps_dprint(sc, MPS_INFO, "%s: unable to find SAS address for handle %d\n", __func__, targ->handle); - mpssas_set_ccbstatus(ccb, CAM_REQ_INVALID); + mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); goto bailout; } mpssas_send_smpcmd(sassc, ccb, sasaddr); @@ -3366,6 +3399,20 @@ mpssas_check_eedp(struct mps_softc *sc, } xpt_path_string(local_path, path_str, sizeof(path_str)); + + /* + * If this is a SATA direct-access end device, + * mark it so that a SCSI StartStopUnit command + * will be sent to it when the driver is being + * shutdown. + */ + if ((cgd.inq_data.device == T_DIRECT) && + (target->devinfo & MPI2_SAS_DEVICE_INFO_SATA_DEVICE) && + ((target->devinfo & MPI2_SAS_DEVICE_INFO_MASK_DEVICE_TYPE) == + MPI2_SAS_DEVICE_INFO_END_DEVICE)) { + lun->stop_at_shutdown = TRUE; + } + mps_dprint(sc, MPS_INFO, "Sending read cap: path %s handle %d\n", path_str, target->handle); Modified: head/sys/dev/mps/mps_sas.h ============================================================================== --- head/sys/dev/mps/mps_sas.h Wed Jul 30 17:57:36 2014 (r269313) +++ head/sys/dev/mps/mps_sas.h Wed Jul 30 17:58:17 2014 (r269314) @@ -35,6 +35,7 @@ struct mpssas_lun { lun_id_t lun_id; uint8_t eedp_formatted; uint32_t eedp_block_size; + uint8_t stop_at_shutdown; }; struct mpssas_target { Modified: head/sys/dev/mps/mps_sas_lsi.c ============================================================================== --- head/sys/dev/mps/mps_sas_lsi.c Wed Jul 30 17:57:36 2014 (r269313) +++ head/sys/dev/mps/mps_sas_lsi.c Wed Jul 30 17:58:17 2014 (r269314) @@ -120,6 +120,9 @@ int mpssas_get_sas_address_for_sata_disk u64 *sas_address, u16 handle, u32 device_info); static int mpssas_volume_add(struct mps_softc *sc, u16 handle); +static void mpssas_SSU_to_SATA_devices(struct mps_softc *sc); +static void mpssas_stop_unit_done(struct cam_periph *periph, + union ccb *done_ccb); void mpssas_evt_handler(struct mps_softc *sc, uintptr_t data, @@ -910,6 +913,138 @@ out: } /** + * mpssas_SSU_to_SATA_devices + * @sc: per adapter object + * + * Looks through the target list and issues a StartStopUnit SCSI command to each + * SATA direct-access device. This helps to ensure that data corruption is + * avoided when the system is being shut down. This must be called after the IR + * System Shutdown RAID Action is sent if in IR mode. + * + * Return nothing. + */ +static void +mpssas_SSU_to_SATA_devices(struct mps_softc *sc) +{ + struct mpssas_softc *sassc = sc->sassc; + union ccb *ccb; + path_id_t pathid = cam_sim_path(sassc->sim); + target_id_t targetid; + struct mpssas_target *target; + struct mpssas_lun *lun; + char path_str[64]; + struct timeval cur_time, start_time; + + /* + * For each LUN of each target, issue a StartStopUnit command to stop + * the device. + */ + sc->SSU_started = TRUE; + sc->SSU_refcount = 0; + for (targetid = 0; targetid < sc->facts->MaxTargets; targetid++) { + target = &sassc->targets[targetid]; + if (target->handle == 0x0) { + continue; + } + + SLIST_FOREACH(lun, &target->luns, lun_link) { + ccb = xpt_alloc_ccb_nowait(); + if (ccb == NULL) { + mps_dprint(sc, MPS_FAULT, "Unable to alloc CCB " + "to stop unit.\n"); + return; + } + + /* + * The stop_at_shutdown flag will be set if this LUN is + * a SATA direct-access end device. + */ + if (lun->stop_at_shutdown) { + if (xpt_create_path(&ccb->ccb_h.path, + xpt_periph, pathid, targetid, + lun->lun_id) != CAM_REQ_CMP) { + mps_dprint(sc, MPS_FAULT, "Unable to " + "create LUN path to stop unit.\n"); + xpt_free_ccb(ccb); + return; + } + xpt_path_string(ccb->ccb_h.path, path_str, + sizeof(path_str)); + + mps_dprint(sc, MPS_INFO, "Sending StopUnit: " + "path %s handle %d\n", path_str, + target->handle); + + /* + * Issue a START STOP UNIT command for the LUN. + * Increment the SSU counter to be used to + * count the number of required replies. + */ + mps_dprint(sc, MPS_INFO, "Incrementing SSU " + "count\n"); + sc->SSU_refcount++; + ccb->ccb_h.target_id = + xpt_path_target_id(ccb->ccb_h.path); + ccb->ccb_h.target_lun = lun->lun_id; + ccb->ccb_h.ppriv_ptr1 = sassc; + scsi_start_stop(&ccb->csio, + /*retries*/0, + mpssas_stop_unit_done, + MSG_SIMPLE_Q_TAG, + /*start*/FALSE, + /*load/eject*/0, + /*immediate*/FALSE, + MPS_SENSE_LEN, + /*timeout*/10000); + xpt_action(ccb); + } + } + } + + /* + * Wait until all of the SSU commands have completed or time has + * expired (60 seconds). pause for 100ms each time through. If any + * command times out, the target will be reset in the SCSI command + * timeout routine. + */ + getmicrotime(&start_time); + while (sc->SSU_refcount) { + pause("mpswait", hz/10); + + getmicrotime(&cur_time); + if ((cur_time.tv_sec - start_time.tv_sec) > 60) { + mps_dprint(sc, MPS_FAULT, "Time has expired waiting " + "for SSU commands to complete.\n"); + break; + } + } +} + +static void +mpssas_stop_unit_done(struct cam_periph *periph, union ccb *done_ccb) +{ + struct mpssas_softc *sassc; + char path_str[64]; + + sassc = (struct mpssas_softc *)done_ccb->ccb_h.ppriv_ptr1; + + xpt_path_string(done_ccb->ccb_h.path, path_str, sizeof(path_str)); + mps_dprint(sassc->sc, MPS_INFO, "Completing stop unit for %s\n", + path_str); + + if (done_ccb == NULL) + return; + + /* + * Nothing more to do except free the CCB and path. If the command + * timed out, an abort reset, then target reset will be issued during + * the SCSI Command process. + */ + xpt_free_path(done_ccb->ccb_h.path); + xpt_free_ccb(done_ccb); +} + +/** * mpssas_ir_shutdown - IR shutdown notification * @sc: per adapter object * @@ -933,7 +1068,7 @@ mpssas_ir_shutdown(struct mps_softc *sc) /* is IR firmware build loaded? */ if (!sc->ir_firmware) - return; + goto out; /* are there any volumes? Look at IR target IDs. */ // TODO-later, this should be looked up in the RAID config structure @@ -958,11 +1093,11 @@ mpssas_ir_shutdown(struct mps_softc *sc) } if (!found_volume) - return; + goto out; if ((cm = mps_alloc_command(sc)) == NULL) { printf("%s: command alloc failed\n", __func__); - return; + goto out; } action = (MPI2_RAID_ACTION_REQUEST *)cm->cm_req; @@ -978,4 +1113,7 @@ mpssas_ir_shutdown(struct mps_softc *sc) */ if (cm) mps_free_command(sc, cm); + +out: + mpssas_SSU_to_SATA_devices(sc); } Modified: head/sys/dev/mps/mpsvar.h ============================================================================== --- head/sys/dev/mps/mpsvar.h Wed Jul 30 17:57:36 2014 (r269313) +++ head/sys/dev/mps/mpsvar.h Wed Jul 30 17:58:17 2014 (r269314) @@ -32,7 +32,7 @@ #ifndef _MPSVAR_H #define _MPSVAR_H -#define MPS_DRIVER_VERSION "16.00.00.00-fbsd" +#define MPS_DRIVER_VERSION "18.00.00.00-fbsd" #define MPS_DB_MAX_WAIT 2500 @@ -417,6 +417,10 @@ struct mps_softc { char exclude_ids[80]; struct timeval lastfail; + + /* StartStopUnit command handling at shutdown */ + uint32_t SSU_refcount; + uint8_t SSU_started; }; struct mps_config_params { @@ -759,6 +763,7 @@ struct mpssas_target * mpssas_find_targe void mpssas_realloc_targets(struct mps_softc *sc, int maxtargets); struct mps_command * mpssas_alloc_tm(struct mps_softc *sc); void mpssas_free_tm(struct mps_softc *sc, struct mps_command *tm); +void mpssas_release_simq_reinit(struct mpssas_softc *sassc); SYSCTL_DECL(_hw_mps);