Date: Tue, 7 Jan 2014 01:51:48 +0000 (UTC) From: Scott Long <scottl@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-10@freebsd.org Subject: svn commit: r260387 - in stable/10/sys: cam cam/ata cam/ctl cam/scsi dev/ahci dev/ata dev/isp dev/mvs dev/siis kern Message-ID: <201401070151.s071pmLP055901@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: scottl Date: Tue Jan 7 01:51:48 2014 New Revision: 260387 URL: http://svnweb.freebsd.org/changeset/base/260387 Log: MFC Alexander Motin's direct dispatch, multi-queue, and finer-grained locking support for CAM r256826: Fix several target mode SIMs to not blindly clear ccb_h.flags field of ATIO CCBs. Not all CCB flags there belong to them. r256836: Remove hard limit on number of BIOs handled with one ATA TRIM request. r256843: Merge CAM locking changes from the projects/camlock branch to radically reduce lock congestion and improve SMP scalability of the SCSI/ATA stack, preparing the ground for the coming next GEOM direct dispatch support. r256888: Unconditionally acquire periph reference on CCB allocation failure. r256895: Fix memory and references leak due to unfreed path. r256960: Move CAM_UNQUEUED_INDEX setting to the last moment and under the periph lock. This fixes race condition with cam_periph_ccbwait(), causing use-after-free. r256975: Minor (mostly cosmetical) addition to r256960. r257054: Some microoptimizations for da and ada drivers: - Replace ordered_tag_count counter with single flag; - From da remove outstanding_cmds counter, duplicating pending_ccbs list; - From da_softc remove unused links field. r257482: Fix lock recursion, triggered by `smartctl -a /dev/adaX`. r257501: Make getenv_*() functions and respectively TUNABLE_*_FETCH() macros not allocate memory and so not require sleepable environment. getenv() has already used on-stack temporary storage, so just use it more rationally. getenv_string() receives buffer as argument, so don't need another one. r257914: Some CAM locks polishing: - Fix LOR and possible lock recursion when handling high-power commands. Introduce new lock to protect left power quota and list of frozen devices. - Correct locking around xpt periph creation. - Remove seems never used XPT_FLAG_OPEN xpt periph flag. Again, Netflix assisted with testing the merge, but all of the credit goes to Alexander and iX Systems. Submitted by: mav Sponsored by: iX Systems Modified: stable/10/sys/cam/ata/ata_da.c stable/10/sys/cam/ata/ata_pmp.c stable/10/sys/cam/ata/ata_xpt.c stable/10/sys/cam/cam_ccb.h stable/10/sys/cam/cam_periph.c stable/10/sys/cam/cam_periph.h stable/10/sys/cam/cam_queue.c stable/10/sys/cam/cam_queue.h stable/10/sys/cam/cam_sim.c stable/10/sys/cam/cam_sim.h stable/10/sys/cam/cam_xpt.c stable/10/sys/cam/cam_xpt.h stable/10/sys/cam/cam_xpt_internal.h stable/10/sys/cam/cam_xpt_sim.h stable/10/sys/cam/ctl/ctl_frontend_cam_sim.c stable/10/sys/cam/ctl/scsi_ctl.c stable/10/sys/cam/scsi/scsi_cd.c stable/10/sys/cam/scsi/scsi_ch.c stable/10/sys/cam/scsi/scsi_da.c stable/10/sys/cam/scsi/scsi_enc.c stable/10/sys/cam/scsi/scsi_enc_internal.h stable/10/sys/cam/scsi/scsi_enc_safte.c stable/10/sys/cam/scsi/scsi_enc_ses.c stable/10/sys/cam/scsi/scsi_pass.c stable/10/sys/cam/scsi/scsi_pt.c stable/10/sys/cam/scsi/scsi_sa.c stable/10/sys/cam/scsi/scsi_sg.c stable/10/sys/cam/scsi/scsi_targ_bh.c stable/10/sys/cam/scsi/scsi_target.c stable/10/sys/cam/scsi/scsi_xpt.c stable/10/sys/dev/ahci/ahci.c stable/10/sys/dev/ahci/ahci.h stable/10/sys/dev/ata/ata-all.c stable/10/sys/dev/isp/isp_freebsd.c stable/10/sys/dev/mvs/mvs.c stable/10/sys/dev/siis/siis.c stable/10/sys/kern/kern_environment.c Directory Properties: stable/10/ (props changed) Modified: stable/10/sys/cam/ata/ata_da.c ============================================================================== --- stable/10/sys/cam/ata/ata_da.c Tue Jan 7 01:40:49 2014 (r260386) +++ stable/10/sys/cam/ata/ata_da.c Tue Jan 7 01:51:48 2014 (r260387) @@ -80,7 +80,7 @@ typedef enum { ADA_FLAG_CAN_NCQ = 0x0008, ADA_FLAG_CAN_DMA = 0x0010, ADA_FLAG_NEED_OTAG = 0x0020, - ADA_FLAG_WENT_IDLE = 0x0040, + ADA_FLAG_WAS_OTAG = 0x0040, ADA_FLAG_CAN_TRIM = 0x0080, ADA_FLAG_OPEN = 0x0100, ADA_FLAG_SCTX_INIT = 0x0200, @@ -103,7 +103,6 @@ typedef enum { ADA_CCB_RAHEAD = 0x01, ADA_CCB_WCACHE = 0x02, ADA_CCB_BUFFER_IO = 0x03, - ADA_CCB_WAITING = 0x04, ADA_CCB_DUMP = 0x05, ADA_CCB_TRIM = 0x06, ADA_CCB_TYPE_MASK = 0x0F, @@ -123,21 +122,20 @@ struct disk_params { #define TRIM_MAX_BLOCKS 8 #define TRIM_MAX_RANGES (TRIM_MAX_BLOCKS * ATA_DSM_BLK_RANGES) -#define TRIM_MAX_BIOS (TRIM_MAX_RANGES * 4) struct trim_request { uint8_t data[TRIM_MAX_RANGES * ATA_DSM_RANGE_SIZE]; - struct bio *bps[TRIM_MAX_BIOS]; + TAILQ_HEAD(, bio) bps; }; struct ada_softc { struct bio_queue_head bio_queue; struct bio_queue_head trim_queue; + int outstanding_cmds; /* Number of active commands */ + int refcount; /* Active xpt_action() calls */ ada_state state; - ada_flags flags; + ada_flags flags; ada_quirks quirks; int sort_io_queue; - int ordered_tag_count; - int outstanding_cmds; int trim_max_ranges; int trim_running; int read_ahead; @@ -630,14 +628,8 @@ adaclose(struct disk *dp) int error; periph = (struct cam_periph *)dp->d_drv1; - cam_periph_lock(periph); - if (cam_periph_hold(periph, PRIBIO) != 0) { - cam_periph_unlock(periph); - cam_periph_release(periph); - return (0); - } - softc = (struct ada_softc *)periph->softc; + cam_periph_lock(periph); CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH, ("adaclose\n")); @@ -645,7 +637,8 @@ adaclose(struct disk *dp) /* We only sync the cache if the drive is capable of it. */ if ((softc->flags & ADA_FLAG_DIRTY) != 0 && (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) != 0 && - (periph->flags & CAM_PERIPH_INVALID) == 0) { + (periph->flags & CAM_PERIPH_INVALID) == 0 && + cam_periph_hold(periph, PRIBIO) == 0) { ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL); cam_fill_ataio(&ccb->ataio, @@ -669,10 +662,13 @@ adaclose(struct disk *dp) else softc->flags &= ~ADA_FLAG_DIRTY; xpt_release_ccb(ccb); + cam_periph_unhold(periph); } softc->flags &= ~ADA_FLAG_OPEN; - cam_periph_unhold(periph); + + while (softc->refcount != 0) + cam_periph_sleep(periph, &softc->refcount, PRIBIO, "adaclose", 1); cam_periph_unlock(periph); cam_periph_release(periph); return (0); @@ -682,23 +678,15 @@ static void adaschedule(struct cam_periph *periph) { struct ada_softc *softc = (struct ada_softc *)periph->softc; - uint32_t prio; if (softc->state != ADA_STATE_NORMAL) return; - /* Check if cam_periph_getccb() was called. */ - prio = periph->immediate_priority; - /* Check if we have more work to do. */ if (bioq_first(&softc->bio_queue) || (!softc->trim_running && bioq_first(&softc->trim_queue))) { - prio = CAM_PRIORITY_NORMAL; + xpt_schedule(periph, CAM_PRIORITY_NORMAL); } - - /* Schedule CCB if any of above is true. */ - if (prio != CAM_PRIORITY_NONE) - xpt_schedule(periph, prio); } /* @@ -962,7 +950,7 @@ adaasync(void *callback_arg, u_int32_t c status = cam_periph_alloc(adaregister, adaoninvalidate, adacleanup, adastart, "ada", CAM_PERIPH_BIO, - cgd->ccb_h.path, adaasync, + path, adaasync, AC_FOUND_DEVICE, cgd); if (status != CAM_REQ_CMP @@ -1038,8 +1026,10 @@ adaasync(void *callback_arg, u_int32_t c softc->state = ADA_STATE_WCACHE; else break; - cam_periph_acquire(periph); - xpt_schedule(periph, CAM_PRIORITY_DEV); + if (cam_periph_acquire(periph) != CAM_REQ_CMP) + softc->state = ADA_STATE_NORMAL; + else + xpt_schedule(periph, CAM_PRIORITY_DEV); } default: cam_periph_async(periph, code, path, arg); @@ -1346,8 +1336,8 @@ adaregister(struct cam_periph *periph, v * Create our sysctl variables, now that we know * we have successfully attached. */ - cam_periph_acquire(periph); - taskqueue_enqueue(taskqueue_thread, &softc->sysctl_task); + if (cam_periph_acquire(periph) == CAM_REQ_CMP) + taskqueue_enqueue(taskqueue_thread, &softc->sysctl_task); /* * Add async callbacks for bus reset and @@ -1365,7 +1355,7 @@ adaregister(struct cam_periph *periph, v * Schedule a periodic event to occasionally send an * ordered tag to a device. */ - callout_init_mtx(&softc->sendordered_c, periph->sim->mtx, 0); + callout_init_mtx(&softc->sendordered_c, cam_periph_mtx(periph), 0); callout_reset(&softc->sendordered_c, (ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL, adasendorderedtag, softc); @@ -1373,16 +1363,17 @@ adaregister(struct cam_periph *periph, v if (ADA_RA >= 0 && cgd->ident_data.support.command1 & ATA_SUPPORT_LOOKAHEAD) { softc->state = ADA_STATE_RAHEAD; - cam_periph_acquire(periph); - xpt_schedule(periph, CAM_PRIORITY_DEV); } else if (ADA_WC >= 0 && cgd->ident_data.support.command1 & ATA_SUPPORT_WRITECACHE) { softc->state = ADA_STATE_WCACHE; - cam_periph_acquire(periph); - xpt_schedule(periph, CAM_PRIORITY_DEV); - } else + } else { softc->state = ADA_STATE_NORMAL; - + return(CAM_REQ_CMP); + } + if (cam_periph_acquire(periph) != CAM_REQ_CMP) + softc->state = ADA_STATE_NORMAL; + else + xpt_schedule(periph, CAM_PRIORITY_DEV); return(CAM_REQ_CMP); } @@ -1400,29 +1391,17 @@ adastart(struct cam_periph *periph, unio struct bio *bp; u_int8_t tag_code; - /* Execute immediate CCB if waiting. */ - if (periph->immediate_priority <= periph->pinfo.priority) { - CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, - ("queuing for immediate ccb\n")); - start_ccb->ccb_h.ccb_state = ADA_CCB_WAITING; - SLIST_INSERT_HEAD(&periph->ccb_list, &start_ccb->ccb_h, - periph_links.sle); - periph->immediate_priority = CAM_PRIORITY_NONE; - wakeup(&periph->ccb_list); - /* Have more work to do, so ensure we stay scheduled */ - adaschedule(periph); - break; - } /* Run TRIM if not running yet. */ if (!softc->trim_running && (bp = bioq_first(&softc->trim_queue)) != 0) { struct trim_request *req = &softc->trim_req; struct bio *bp1; uint64_t lastlba = (uint64_t)-1; - int bps = 0, c, lastcount = 0, off, ranges = 0; + int c, lastcount = 0, off, ranges = 0; softc->trim_running = 1; bzero(req, sizeof(*req)); + TAILQ_INIT(&req->bps); bp1 = bp; do { uint64_t lba = bp1->bio_pblkno; @@ -1465,10 +1444,9 @@ adastart(struct cam_periph *periph, unio */ } lastlba = lba; - req->bps[bps++] = bp1; + TAILQ_INSERT_TAIL(&req->bps, bp1, bio_queue); bp1 = bioq_first(&softc->trim_queue); - if (bps >= TRIM_MAX_BIOS || - bp1 == NULL || + if (bp1 == NULL || bp1->bio_bcount / softc->params.secsize > (softc->trim_max_ranges - ranges) * ATA_DSM_RANGE_MAX) @@ -1487,6 +1465,7 @@ adastart(struct cam_periph *periph, unio ATA_DSM_TRIM, 0, (ranges + ATA_DSM_BLK_RANGES - 1) / ATA_DSM_BLK_RANGES); start_ccb->ccb_h.ccb_state = ADA_CCB_TRIM; + start_ccb->ccb_h.flags |= CAM_UNLOCKED; goto out; } /* Run regular command. */ @@ -1500,7 +1479,7 @@ adastart(struct cam_periph *periph, unio if ((bp->bio_flags & BIO_ORDERED) != 0 || (softc->flags & ADA_FLAG_NEED_OTAG) != 0) { softc->flags &= ~ADA_FLAG_NEED_OTAG; - softc->ordered_tag_count++; + softc->flags |= ADA_FLAG_WAS_OTAG; tag_code = 0; } else { tag_code = 1; @@ -1655,10 +1634,15 @@ adastart(struct cam_periph *periph, unio break; } start_ccb->ccb_h.ccb_state = ADA_CCB_BUFFER_IO; + start_ccb->ccb_h.flags |= CAM_UNLOCKED; out: start_ccb->ccb_h.ccb_bp = bp; softc->outstanding_cmds++; + softc->refcount++; + cam_periph_unlock(periph); xpt_action(start_ccb); + cam_periph_lock(periph); + softc->refcount--; /* May have more work to do, so ensure we stay scheduled */ adaschedule(periph); @@ -1667,13 +1651,6 @@ out: case ADA_STATE_RAHEAD: case ADA_STATE_WCACHE: { - if ((periph->flags & CAM_PERIPH_INVALID) != 0) { - softc->state = ADA_STATE_NORMAL; - xpt_release_ccb(start_ccb); - cam_periph_release_locked(periph); - return; - } - cam_fill_ataio(ataio, 1, adadone, @@ -1722,10 +1699,12 @@ adadone(struct cam_periph *periph, union struct bio *bp; int error; + cam_periph_lock(periph); if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) { error = adaerror(done_ccb, 0, 0); if (error == ERESTART) { /* A retry was scheduled, so just return. */ + cam_periph_unlock(periph); return; } if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) @@ -1754,29 +1733,32 @@ adadone(struct cam_periph *periph, union } softc->outstanding_cmds--; if (softc->outstanding_cmds == 0) - softc->flags |= ADA_FLAG_WENT_IDLE; + softc->flags |= ADA_FLAG_WAS_OTAG; + xpt_release_ccb(done_ccb); if (state == ADA_CCB_TRIM) { - struct trim_request *req = - (struct trim_request *)ataio->data_ptr; - int i; - - for (i = 1; i < TRIM_MAX_BIOS && req->bps[i]; i++) { - struct bio *bp1 = req->bps[i]; + TAILQ_HEAD(, bio) queue; + struct bio *bp1; - bp1->bio_error = bp->bio_error; - if (bp->bio_flags & BIO_ERROR) { + TAILQ_INIT(&queue); + TAILQ_CONCAT(&queue, &softc->trim_req.bps, bio_queue); + softc->trim_running = 0; + adaschedule(periph); + cam_periph_unlock(periph); + while ((bp1 = TAILQ_FIRST(&queue)) != NULL) { + TAILQ_REMOVE(&queue, bp1, bio_queue); + bp1->bio_error = error; + if (error != 0) { bp1->bio_flags |= BIO_ERROR; bp1->bio_resid = bp1->bio_bcount; } else bp1->bio_resid = 0; biodone(bp1); } - softc->trim_running = 0; - biodone(bp); - adaschedule(periph); - } else + } else { + cam_periph_unlock(periph); biodone(bp); - break; + } + return; } case ADA_CCB_RAHEAD: { @@ -1852,12 +1834,6 @@ out: cam_periph_release_locked(periph); return; } - case ADA_CCB_WAITING: - { - /* Caller will release the CCB */ - wakeup(&done_ccb->ccb_h.cbfcnp); - return; - } case ADA_CCB_DUMP: /* No-op. We're polling */ return; @@ -1919,14 +1895,11 @@ adasendorderedtag(void *arg) struct ada_softc *softc = arg; if (ada_send_ordered) { - if ((softc->ordered_tag_count == 0) - && ((softc->flags & ADA_FLAG_WENT_IDLE) == 0)) { - softc->flags |= ADA_FLAG_NEED_OTAG; + if (softc->outstanding_cmds > 0) { + if ((softc->flags & ADA_FLAG_WAS_OTAG) == 0) + softc->flags |= ADA_FLAG_NEED_OTAG; + softc->flags &= ~ADA_FLAG_WAS_OTAG; } - if (softc->outstanding_cmds > 0) - softc->flags &= ~ADA_FLAG_WENT_IDLE; - - softc->ordered_tag_count = 0; } /* Queue us up again */ callout_reset(&softc->sendordered_c, Modified: stable/10/sys/cam/ata/ata_pmp.c ============================================================================== --- stable/10/sys/cam/ata/ata_pmp.c Tue Jan 7 01:40:49 2014 (r260386) +++ stable/10/sys/cam/ata/ata_pmp.c Tue Jan 7 01:51:48 2014 (r260387) @@ -293,7 +293,7 @@ pmpasync(void *callback_arg, u_int32_t c status = cam_periph_alloc(pmpregister, pmponinvalidate, pmpcleanup, pmpstart, "pmp", CAM_PERIPH_BIO, - cgd->ccb_h.path, pmpasync, + path, pmpasync, AC_FOUND_DEVICE, cgd); if (status != CAM_REQ_CMP @@ -318,13 +318,17 @@ pmpasync(void *callback_arg, u_int32_t c if (code == AC_SENT_BDR || code == AC_BUS_RESET) softc->found = 0; /* We have to reset everything. */ if (softc->state == PMP_STATE_NORMAL) { - if (softc->pm_pid == 0x37261095 || - softc->pm_pid == 0x38261095) - softc->state = PMP_STATE_PM_QUIRKS_1; - else - softc->state = PMP_STATE_PRECONFIG; - cam_periph_acquire(periph); - xpt_schedule(periph, CAM_PRIORITY_DEV); + if (cam_periph_acquire(periph) == CAM_REQ_CMP) { + if (softc->pm_pid == 0x37261095 || + softc->pm_pid == 0x38261095) + softc->state = PMP_STATE_PM_QUIRKS_1; + else + softc->state = PMP_STATE_PRECONFIG; + xpt_schedule(periph, CAM_PRIORITY_DEV); + } else { + pmprelease(periph, softc->found); + xpt_release_boot(); + } } else softc->restart = 1; break; Modified: stable/10/sys/cam/ata/ata_xpt.c ============================================================================== --- stable/10/sys/cam/ata/ata_xpt.c Tue Jan 7 01:40:49 2014 (r260386) +++ stable/10/sys/cam/ata/ata_xpt.c Tue Jan 7 01:51:48 2014 (r260387) @@ -182,7 +182,7 @@ static struct cam_ed * static void ata_device_transport(struct cam_path *path); static void ata_get_transfer_settings(struct ccb_trans_settings *cts); static void ata_set_transfer_settings(struct ccb_trans_settings *cts, - struct cam_ed *device, + struct cam_path *path, int async_update); static void ata_dev_async(u_int32_t async_code, struct cam_eb *bus, @@ -249,6 +249,7 @@ proberegister(struct cam_periph *periph, return (status); } CAM_DEBUG(periph->path, CAM_DEBUG_PROBE, ("Probe started\n")); + ata_device_transport(periph->path); probeschedule(periph); return(CAM_REQ_CMP); } @@ -1320,6 +1321,7 @@ ata_scan_bus(struct cam_periph *periph, struct cam_path *path; ata_scan_bus_info *scan_info; union ccb *work_ccb, *reset_ccb; + struct mtx *mtx; cam_status status; CAM_DEBUG(request_ccb->ccb_h.path, CAM_DEBUG_TRACE, @@ -1395,11 +1397,14 @@ ata_scan_bus(struct cam_periph *periph, xpt_done(request_ccb); break; } + mtx = xpt_path_mtx(scan_info->request_ccb->ccb_h.path); goto scan_next; case XPT_SCAN_LUN: work_ccb = request_ccb; /* Reuse the same CCB to query if a device was really found */ scan_info = (ata_scan_bus_info *)work_ccb->ccb_h.ppriv_ptr0; + mtx = xpt_path_mtx(scan_info->request_ccb->ccb_h.path); + mtx_lock(mtx); /* If there is PMP... */ if ((scan_info->cpi->hba_inquiry & PI_SATAPM) && (scan_info->counter == scan_info->cpi->max_target)) { @@ -1428,6 +1433,7 @@ ata_scan_bus(struct cam_periph *periph, ((scan_info->cpi->hba_inquiry & PI_SATAPM) ? 0 : scan_info->cpi->max_target)) { done: + mtx_unlock(mtx); xpt_free_ccb(work_ccb); xpt_free_ccb((union ccb *)scan_info->cpi); request_ccb = scan_info->request_ccb; @@ -1444,6 +1450,8 @@ scan_next: scan_info->request_ccb->ccb_h.path_id, scan_info->counter, 0); if (status != CAM_REQ_CMP) { + if (request_ccb->ccb_h.func_code == XPT_SCAN_LUN) + mtx_unlock(mtx); printf("xpt_scan_bus: xpt_create_path failed" " with status %#x, bus scan halted\n", status); @@ -1459,9 +1467,15 @@ scan_next: scan_info->request_ccb->ccb_h.pinfo.priority); work_ccb->ccb_h.func_code = XPT_SCAN_LUN; work_ccb->ccb_h.cbfcnp = ata_scan_bus; + work_ccb->ccb_h.flags |= CAM_UNLOCKED; work_ccb->ccb_h.ppriv_ptr0 = scan_info; work_ccb->crcn.flags = scan_info->request_ccb->crcn.flags; + mtx_unlock(mtx); + if (request_ccb->ccb_h.func_code == XPT_SCAN_LUN) + mtx = NULL; xpt_action(work_ccb); + if (mtx != NULL) + mtx_lock(mtx); break; default: break; @@ -1476,6 +1490,7 @@ ata_scan_lun(struct cam_periph *periph, cam_status status; struct cam_path *new_path; struct cam_periph *old_periph; + int lock; CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_scan_lun\n")); @@ -1510,10 +1525,14 @@ ata_scan_lun(struct cam_periph *periph, } xpt_setup_ccb(&request_ccb->ccb_h, new_path, CAM_PRIORITY_XPT); request_ccb->ccb_h.cbfcnp = xptscandone; + request_ccb->ccb_h.flags |= CAM_UNLOCKED; request_ccb->ccb_h.func_code = XPT_SCAN_LUN; request_ccb->crcn.flags = flags; } + lock = (xpt_path_owned(path) == 0); + if (lock) + xpt_path_lock(path); if ((old_periph = cam_periph_find(path, "aprobe")) != NULL) { if ((old_periph->flags & CAM_PERIPH_INVALID) == 0) { probe_softc *softc; @@ -1540,6 +1559,8 @@ ata_scan_lun(struct cam_periph *periph, xpt_done(request_ccb); } } + if (lock) + xpt_path_unlock(path); } static void @@ -1553,7 +1574,6 @@ xptscandone(struct cam_periph *periph, u static struct cam_ed * ata_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id) { - struct cam_path path; struct ata_quirk_entry *quirk; struct cam_ed *device; @@ -1574,22 +1594,6 @@ ata_alloc_device(struct cam_eb *bus, str device->queue_flags = 0; device->serial_num = NULL; device->serial_num_len = 0; - - /* - * XXX should be limited by number of CCBs this bus can - * do. - */ - bus->sim->max_ccbs += device->ccbq.devq_openings; - if (lun_id != CAM_LUN_WILDCARD) { - xpt_compile_path(&path, - NULL, - bus->path_id, - target->target_id, - lun_id); - ata_device_transport(&path); - xpt_release_path(&path); - } - return (device); } @@ -1712,15 +1716,8 @@ ata_dev_advinfo(union ccb *start_ccb) start_ccb->ccb_h.status = CAM_REQ_CMP; if (cdai->flags & CDAI_FLAG_STORE) { - int owned; - - owned = mtx_owned(start_ccb->ccb_h.path->bus->sim->mtx); - if (owned == 0) - mtx_lock(start_ccb->ccb_h.path->bus->sim->mtx); xpt_async(AC_ADVINFO_CHANGED, start_ccb->ccb_h.path, (void *)(uintptr_t)cdai->buftype); - if (owned == 0) - mtx_unlock(start_ccb->ccb_h.path->bus->sim->mtx); } } @@ -1732,7 +1729,7 @@ ata_action(union ccb *start_ccb) case XPT_SET_TRAN_SETTINGS: { ata_set_transfer_settings(&start_ccb->cts, - start_ccb->ccb_h.path->device, + start_ccb->ccb_h.path, /*async_update*/FALSE); break; } @@ -1791,11 +1788,9 @@ ata_get_transfer_settings(struct ccb_tra struct ccb_trans_settings_ata *ata; struct ccb_trans_settings_scsi *scsi; struct cam_ed *device; - struct cam_sim *sim; device = cts->ccb_h.path->device; - sim = cts->ccb_h.path->bus->sim; - (*(sim->sim_action))(sim, (union ccb *)cts); + xpt_action_default((union ccb *)cts); if (cts->protocol == PROTO_UNKNOWN || cts->protocol == PROTO_UNSPECIFIED) { @@ -1832,17 +1827,17 @@ ata_get_transfer_settings(struct ccb_tra } static void -ata_set_transfer_settings(struct ccb_trans_settings *cts, struct cam_ed *device, +ata_set_transfer_settings(struct ccb_trans_settings *cts, struct cam_path *path, int async_update) { struct ccb_pathinq cpi; struct ccb_trans_settings_ata *ata; struct ccb_trans_settings_scsi *scsi; - struct cam_sim *sim; struct ata_params *ident_data; struct scsi_inquiry_data *inq_data; + struct cam_ed *device; - if (device == NULL) { + if (path == NULL || (device = path->device) == NULL) { cts->ccb_h.status = CAM_PATH_INVALID; xpt_done((union ccb *)cts); return; @@ -1859,14 +1854,14 @@ ata_set_transfer_settings(struct ccb_tra cts->protocol_version = device->protocol_version; if (cts->protocol != device->protocol) { - xpt_print(cts->ccb_h.path, "Uninitialized Protocol %x:%x?\n", + xpt_print(path, "Uninitialized Protocol %x:%x?\n", cts->protocol, device->protocol); cts->protocol = device->protocol; } if (cts->protocol_version > device->protocol_version) { if (bootverbose) { - xpt_print(cts->ccb_h.path, "Down reving Protocol " + xpt_print(path, "Down reving Protocol " "Version from %d to %d?\n", cts->protocol_version, device->protocol_version); } @@ -1884,21 +1879,20 @@ ata_set_transfer_settings(struct ccb_tra cts->transport_version = device->transport_version; if (cts->transport != device->transport) { - xpt_print(cts->ccb_h.path, "Uninitialized Transport %x:%x?\n", + xpt_print(path, "Uninitialized Transport %x:%x?\n", cts->transport, device->transport); cts->transport = device->transport; } if (cts->transport_version > device->transport_version) { if (bootverbose) { - xpt_print(cts->ccb_h.path, "Down reving Transport " + xpt_print(path, "Down reving Transport " "Version from %d to %d?\n", cts->transport_version, device->transport_version); } cts->transport_version = device->transport_version; } - sim = cts->ccb_h.path->bus->sim; ident_data = &device->ident_data; inq_data = &device->inq_data; if (cts->protocol == PROTO_ATA) @@ -1909,7 +1903,7 @@ ata_set_transfer_settings(struct ccb_tra scsi = &cts->proto_specific.scsi; else scsi = NULL; - xpt_setup_ccb(&cpi.ccb_h, cts->ccb_h.path, CAM_PRIORITY_NONE); + xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NONE); cpi.ccb_h.func_code = XPT_PATH_INQ; xpt_action((union ccb *)&cpi); @@ -1953,11 +1947,11 @@ ata_set_transfer_settings(struct ccb_tra device->tag_delay_count = CAM_TAG_DELAY_COUNT; device->flags |= CAM_DEV_TAG_AFTER_COUNT; } else if (nowt && !newt) - xpt_stop_tags(cts->ccb_h.path); + xpt_stop_tags(path); } if (async_update == FALSE) - (*(sim->sim_action))(sim, (union ccb *)cts); + xpt_action_default((union ccb *)cts); } /* @@ -2014,10 +2008,14 @@ ata_dev_async(u_int32_t async_code, stru xpt_release_device(device); } else if (async_code == AC_TRANSFER_NEG) { struct ccb_trans_settings *settings; + struct cam_path path; settings = (struct ccb_trans_settings *)async_arg; - ata_set_transfer_settings(settings, device, + xpt_compile_path(&path, NULL, bus->path_id, target->target_id, + device->lun_id); + ata_set_transfer_settings(settings, &path, /*async_update*/TRUE); + xpt_release_path(&path); } } @@ -2030,7 +2028,7 @@ ata_announce_periph(struct cam_periph *p u_int speed; u_int mb; - mtx_assert(periph->sim->mtx, MA_OWNED); + cam_periph_assert(periph, MA_OWNED); xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NORMAL); cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS; Modified: stable/10/sys/cam/cam_ccb.h ============================================================================== --- stable/10/sys/cam/cam_ccb.h Tue Jan 7 01:40:49 2014 (r260386) +++ stable/10/sys/cam/cam_ccb.h Tue Jan 7 01:51:48 2014 (r260387) @@ -104,7 +104,9 @@ typedef enum { CAM_SEND_SENSE = 0x08000000,/* Send sense data with status */ CAM_TERM_IO = 0x10000000,/* Terminate I/O Message sup. */ CAM_DISCONNECT = 0x20000000,/* Disconnects are mandatory */ - CAM_SEND_STATUS = 0x40000000 /* Send status after data phase */ + CAM_SEND_STATUS = 0x40000000,/* Send status after data phase */ + + CAM_UNLOCKED = 0x80000000 /* Call callback without lock. */ } ccb_flags; typedef enum { @@ -151,6 +153,9 @@ typedef enum { /* Device statistics (error counts, etc.) */ XPT_DEV_ADVINFO = 0x0e, /* Get/Set Device advanced information */ + XPT_ASYNC = 0x0f | XPT_FC_QUEUED | XPT_FC_USER_CCB + | XPT_FC_XPT_ONLY, + /* Asynchronous event */ /* SCSI Control Functions: 0x10->0x1F */ XPT_ABORT = 0x10, /* Abort the specified CCB */ @@ -1154,6 +1159,16 @@ struct ccb_dev_advinfo { }; /* + * CCB for sending async events + */ +struct ccb_async { + struct ccb_hdr ccb_h; + uint32_t async_code; + off_t async_arg_size; + void *async_arg_ptr; +}; + +/* * Union of all CCB types for kernel space allocation. This union should * never be used for manipulating CCBs - its only use is for the allocation * and deallocation of raw CCB space and is the return type of xpt_ccb_alloc @@ -1192,6 +1207,7 @@ union ccb { struct ccb_debug cdbg; struct ccb_ataio ataio; struct ccb_dev_advinfo cdai; + struct ccb_async casync; }; __BEGIN_DECLS Modified: stable/10/sys/cam/cam_periph.c ============================================================================== --- stable/10/sys/cam/cam_periph.c Tue Jan 7 01:40:49 2014 (r260386) +++ stable/10/sys/cam/cam_periph.c Tue Jan 7 01:51:48 2014 (r260387) @@ -196,12 +196,12 @@ cam_periph_alloc(periph_ctor_t *periph_c path_id = xpt_path_path_id(path); target_id = xpt_path_target_id(path); lun_id = xpt_path_lun_id(path); - cam_init_pinfo(&periph->pinfo); periph->periph_start = periph_start; periph->periph_dtor = periph_dtor; periph->periph_oninval = periph_oninvalidate; periph->type = type; periph->periph_name = name; + periph->scheduled_priority = CAM_PRIORITY_NONE; periph->immediate_priority = CAM_PRIORITY_NONE; periph->refcount = 1; /* Dropped by invalidation. */ periph->sim = sim; @@ -298,7 +298,7 @@ cam_periph_find(struct cam_path *path, c TAILQ_FOREACH(periph, &(*p_drv)->units, unit_links) { if (xpt_path_comp(periph->path, path) == 0) { xpt_unlock_buses(); - mtx_assert(periph->sim->mtx, MA_OWNED); + cam_periph_assert(periph, MA_OWNED); return(periph); } } @@ -379,7 +379,7 @@ void cam_periph_release_locked_buses(struct cam_periph *periph) { - mtx_assert(periph->sim->mtx, MA_OWNED); + cam_periph_assert(periph, MA_OWNED); KASSERT(periph->refcount >= 1, ("periph->refcount >= 1")); if (--periph->refcount == 0) camperiphfree(periph); @@ -400,16 +400,16 @@ cam_periph_release_locked(struct cam_per void cam_periph_release(struct cam_periph *periph) { - struct cam_sim *sim; + struct mtx *mtx; if (periph == NULL) return; - sim = periph->sim; - mtx_assert(sim->mtx, MA_NOTOWNED); - mtx_lock(sim->mtx); + cam_periph_assert(periph, MA_NOTOWNED); + mtx = cam_periph_mtx(periph); + mtx_lock(mtx); cam_periph_release_locked(periph); - mtx_unlock(sim->mtx); + mtx_unlock(mtx); } int @@ -427,10 +427,10 @@ cam_periph_hold(struct cam_periph *perip if (cam_periph_acquire(periph) != CAM_REQ_CMP) return (ENXIO); - mtx_assert(periph->sim->mtx, MA_OWNED); + cam_periph_assert(periph, MA_OWNED); while ((periph->flags & CAM_PERIPH_LOCKED) != 0) { periph->flags |= CAM_PERIPH_LOCK_WANTED; - if ((error = mtx_sleep(periph, periph->sim->mtx, priority, + if ((error = cam_periph_sleep(periph, periph, priority, "caplck", 0)) != 0) { cam_periph_release_locked(periph); return (error); @@ -449,7 +449,7 @@ void cam_periph_unhold(struct cam_periph *periph) { - mtx_assert(periph->sim->mtx, MA_OWNED); + cam_periph_assert(periph, MA_OWNED); periph->flags &= ~CAM_PERIPH_LOCKED; if ((periph->flags & CAM_PERIPH_LOCK_WANTED) != 0) { @@ -577,7 +577,7 @@ void cam_periph_invalidate(struct cam_periph *periph) { - mtx_assert(periph->sim->mtx, MA_OWNED); + cam_periph_assert(periph, MA_OWNED); /* * We only call this routine the first time a peripheral is * invalidated. @@ -600,7 +600,9 @@ camperiphfree(struct cam_periph *periph) { struct periph_driver **p_drv; - mtx_assert(periph->sim->mtx, MA_OWNED); + cam_periph_assert(periph, MA_OWNED); + KASSERT(periph->periph_allocating == 0, ("%s%d: freed while allocating", + periph->periph_name, periph->unit_number)); for (p_drv = periph_drivers; *p_drv != NULL; p_drv++) { if (strcmp((*p_drv)->driver_name, periph->periph_name) == 0) break; @@ -947,40 +949,14 @@ cam_periph_unmapmem(union ccb *ccb, stru PRELE(curproc); } -union ccb * -cam_periph_getccb(struct cam_periph *periph, u_int32_t priority) -{ - struct ccb_hdr *ccb_h; - - mtx_assert(periph->sim->mtx, MA_OWNED); - CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("entering cdgetccb\n")); - - while (SLIST_FIRST(&periph->ccb_list) == NULL) { - if (periph->immediate_priority > priority) - periph->immediate_priority = priority; - xpt_schedule(periph, priority); - if ((SLIST_FIRST(&periph->ccb_list) != NULL) - && (SLIST_FIRST(&periph->ccb_list)->pinfo.priority == priority)) - break; - mtx_assert(periph->sim->mtx, MA_OWNED); - mtx_sleep(&periph->ccb_list, periph->sim->mtx, PRIBIO, "cgticb", - 0); - } - - ccb_h = SLIST_FIRST(&periph->ccb_list); - SLIST_REMOVE_HEAD(&periph->ccb_list, periph_links.sle); - return ((union ccb *)ccb_h); -} - void cam_periph_ccbwait(union ccb *ccb) { - struct cam_sim *sim; - sim = xpt_path_sim(ccb->ccb_h.path); if ((ccb->ccb_h.pinfo.index != CAM_UNQUEUED_INDEX) || ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_INPROG)) - mtx_sleep(&ccb->ccb_h.cbfcnp, sim->mtx, PRIBIO, "cbwait", 0); + xpt_path_sleep(ccb->ccb_h.path, &ccb->ccb_h.cbfcnp, PRIBIO, + "cbwait", 0); } int @@ -1045,6 +1021,14 @@ cam_periph_ioctl(struct cam_periph *peri return(error); } +static void +cam_periph_done(struct cam_periph *periph, union ccb *done_ccb) +{ + + /* Caller will release the CCB */ + wakeup(&done_ccb->ccb_h.cbfcnp); +} + int cam_periph_runccb(union ccb *ccb, int (*error_routine)(union ccb *ccb, @@ -1053,12 +1037,9 @@ cam_periph_runccb(union ccb *ccb, cam_flags camflags, u_int32_t sense_flags, struct devstat *ds) { - struct cam_sim *sim; int error; - error = 0; - sim = xpt_path_sim(ccb->ccb_h.path); - mtx_assert(sim->mtx, MA_OWNED); + xpt_path_assert(ccb->ccb_h.path, MA_OWNED); /* * If the user has supplied a stats structure, and if we understand @@ -1068,6 +1049,7 @@ cam_periph_runccb(union ccb *ccb, ccb->ccb_h.func_code == XPT_ATA_IO)) devstat_start_transaction(ds, NULL); + ccb->ccb_h.cbfcnp = cam_periph_done; xpt_action(ccb); do { @@ -1786,9 +1768,11 @@ cam_periph_error(union ccb *ccb, cam_fla scan_ccb->ccb_h.func_code = XPT_SCAN_TGT; scan_ccb->crcn.flags = 0; xpt_rescan(scan_ccb); - } else + } else { xpt_print(newpath, "Can't allocate CCB to rescan target\n"); + xpt_free_path(newpath); + } } } Modified: stable/10/sys/cam/cam_periph.h ============================================================================== --- stable/10/sys/cam/cam_periph.h Tue Jan 7 01:40:49 2014 (r260386) +++ stable/10/sys/cam/cam_periph.h Tue Jan 7 01:51:48 2014 (r260387) @@ -35,6 +35,7 @@ #include <cam/cam_sim.h> #ifdef _KERNEL +#include <sys/taskqueue.h> #include <cam/cam_xpt.h> @@ -103,7 +104,6 @@ typedef cam_status periph_ctor_t (struct typedef void periph_oninv_t (struct cam_periph *periph); typedef void periph_dtor_t (struct cam_periph *periph); struct cam_periph { - cam_pinfo pinfo; periph_start_t *periph_start; periph_oninv_t *periph_oninval; periph_dtor_t *periph_dtor; @@ -120,15 +120,20 @@ struct cam_periph { #define CAM_PERIPH_INVALID 0x08 #define CAM_PERIPH_NEW_DEV_FOUND 0x10 #define CAM_PERIPH_RECOVERY_INPROG 0x20 +#define CAM_PERIPH_RUN_TASK 0x40 #define CAM_PERIPH_FREE 0x80 #define CAM_PERIPH_ANNOUNCED 0x100 - u_int32_t immediate_priority; + uint32_t scheduled_priority; + uint32_t immediate_priority; + int periph_allocating; + int periph_allocated; u_int32_t refcount; SLIST_HEAD(, ccb_hdr) ccb_list; /* For "immediate" requests */ SLIST_ENTRY(cam_periph) periph_links; TAILQ_ENTRY(cam_periph) unit_links; ac_callback_t *deferred_callback; ac_code deferred_ac; + struct task periph_run_task; }; #define CAM_PERIPH_MAXMAPS 2 @@ -185,30 +190,26 @@ void cam_periph_freeze_after_event(stru int cam_periph_error(union ccb *ccb, cam_flags camflags, u_int32_t sense_flags, union ccb *save_ccb); -static __inline void -cam_periph_lock(struct cam_periph *periph) +static __inline struct mtx * +cam_periph_mtx(struct cam_periph *periph) { - mtx_lock(periph->sim->mtx); + return (xpt_path_mtx(periph->path)); } -static __inline void -cam_periph_unlock(struct cam_periph *periph) -{ - mtx_unlock(periph->sim->mtx); -} +#define cam_periph_owned(periph) \ + mtx_owned(xpt_path_mtx((periph)->path)) -static __inline int -cam_periph_owned(struct cam_periph *periph) -{ - return (mtx_owned(periph->sim->mtx)); -} +#define cam_periph_lock(periph) \ + mtx_lock(xpt_path_mtx((periph)->path)) -static __inline int -cam_periph_sleep(struct cam_periph *periph, void *chan, int priority, - const char *wmesg, int timo) -{ - return (msleep(chan, periph->sim->mtx, priority, wmesg, timo)); -} +#define cam_periph_unlock(periph) \ + mtx_unlock(xpt_path_mtx((periph)->path)) + *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201401070151.s071pmLP055901>