From owner-dev-commits-src-all@freebsd.org Sun Feb 21 21:52:38 2021 Return-Path: Delivered-To: dev-commits-src-all@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id 62EE653D548; Sun, 21 Feb 2021 21:52:38 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256 client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "R3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4DkJvV2KZYz3lLR; Sun, 21 Feb 2021 21:52:38 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org (gitrepo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:5]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 428391A2B5; Sun, 21 Feb 2021 21:52:38 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org ([127.0.1.44]) by gitrepo.freebsd.org (8.16.1/8.16.1) with ESMTP id 11LLqchs047424; Sun, 21 Feb 2021 21:52:38 GMT (envelope-from git@gitrepo.freebsd.org) Received: (from git@localhost) by gitrepo.freebsd.org (8.16.1/8.16.1/Submit) id 11LLqc3Z047423; Sun, 21 Feb 2021 21:52:38 GMT (envelope-from git) Date: Sun, 21 Feb 2021 21:52:38 GMT Message-Id: <202102212152.11LLqc3Z047423@gitrepo.freebsd.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org From: Alexander Motin Subject: git: 2c7dc6bae9fd - main - Refactor CTL datamove KPI. MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit X-Git-Committer: mav X-Git-Repository: src X-Git-Refname: refs/heads/main X-Git-Reftype: branch X-Git-Commit: 2c7dc6bae9fd5c2fa0a65768df8e4e99c2f159f1 Auto-Submitted: auto-generated X-BeenThere: dev-commits-src-all@freebsd.org X-Mailman-Version: 2.1.34 Precedence: list List-Id: Commit messages for all branches of the src repository List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 21 Feb 2021 21:52:38 -0000 The branch main has been updated by mav: URL: https://cgit.FreeBSD.org/src/commit/?id=2c7dc6bae9fd5c2fa0a65768df8e4e99c2f159f1 commit 2c7dc6bae9fd5c2fa0a65768df8e4e99c2f159f1 Author: Alexander Motin AuthorDate: 2021-02-21 21:45:14 +0000 Commit: Alexander Motin CommitDate: 2021-02-21 21:52:33 +0000 Refactor CTL datamove KPI. - Make frontends call unified CTL core method ctl_datamove_done() to report move completion. It allows to reduce code duplication in differerent backends by accounting DMA time in common code. - Add to ctl_datamove_done() and be_move_done() callback samethr argument, reporting whether the callback is called in the same context as ctl_datamove(). It allows for some cases like iSCSI write with immediate data or camsim frontend write save one context switch, since we know that the context is sleepable. - Remove data_move_done() methods from struct ctl_backend_driver, unused since forever. MFC after: 1 month --- sys/cam/ctl/ctl.c | 112 +++++++++++++++++-------------------- sys/cam/ctl/ctl.h | 3 +- sys/cam/ctl/ctl_backend.h | 1 - sys/cam/ctl/ctl_backend_block.c | 64 ++++++--------------- sys/cam/ctl/ctl_backend_ramdisk.c | 36 ++---------- sys/cam/ctl/ctl_frontend_cam_sim.c | 2 +- sys/cam/ctl/ctl_frontend_ioctl.c | 2 +- sys/cam/ctl/ctl_frontend_iscsi.c | 23 ++++---- sys/cam/ctl/ctl_io.h | 2 +- sys/cam/ctl/ctl_tpc_local.c | 2 +- sys/cam/ctl/scsi_ctl.c | 3 +- sys/dev/usb/storage/cfumass.c | 6 +- 12 files changed, 94 insertions(+), 162 deletions(-) diff --git a/sys/cam/ctl/ctl.c b/sys/cam/ctl/ctl.c index 464fd8dfc25e..9a9ae70c2173 100644 --- a/sys/cam/ctl/ctl.c +++ b/sys/cam/ctl/ctl.c @@ -535,9 +535,9 @@ static void ctl_done_timer_wakeup(void *arg); static void ctl_send_datamove_done(union ctl_io *io, int have_lock); static void ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq); -static int ctl_datamove_remote_dm_write_cb(union ctl_io *io); +static int ctl_datamove_remote_dm_write_cb(union ctl_io *io, bool samethr); static void ctl_datamove_remote_write(union ctl_io *io); -static int ctl_datamove_remote_dm_read_cb(union ctl_io *io); +static int ctl_datamove_remote_dm_read_cb(union ctl_io *io, bool samethr); static void ctl_datamove_remote_read_cb(struct ctl_ha_dt_req *rq); static int ctl_datamove_remote_sgl_setup(union ctl_io *io); static int ctl_datamove_remote_xfer(union ctl_io *io, unsigned command, @@ -736,7 +736,7 @@ ctl_ha_datamove(union ctl_io *io) sizeof(struct ctl_sg_entry) * msg.dt.cur_sg_entries, M_WAITOK) > CTL_HA_STATUS_SUCCESS) { io->io_hdr.port_status = 31341; - io->scsiio.be_move_done(io); + ctl_datamove_done(io, true); return; } msg.dt.sent_sg_entries = sg_entries_sent; @@ -753,7 +753,7 @@ ctl_ha_datamove(union ctl_io *io) if (lun) mtx_unlock(&lun->lun_lock); io->io_hdr.port_status = 31342; - io->scsiio.be_move_done(io); + ctl_datamove_done(io, true); return; } io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE; @@ -5028,7 +5028,7 @@ ctl_lun_capacity_changed(struct ctl_be_lun *be_lun) * make it down to say RAIDCore's configuration code. */ int -ctl_config_move_done(union ctl_io *io) +ctl_config_move_done(union ctl_io *io, bool samethr) { int retval; @@ -5036,18 +5036,6 @@ ctl_config_move_done(union ctl_io *io) KASSERT(io->io_hdr.io_type == CTL_IO_SCSI, ("%s: unexpected I/O type %x", __func__, io->io_hdr.io_type)); - if ((io->io_hdr.port_status != 0) && - ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE || - (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) { - ctl_set_internal_failure(&io->scsiio, /*sks_valid*/ 1, - /*retry_count*/ io->io_hdr.port_status); - } else if (io->scsiio.kern_data_resid != 0 && - (io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT && - ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE || - (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) { - ctl_set_invalid_field_ciu(&io->scsiio); - } - if (ctl_debug & CTL_DEBUG_CDB_DATA) ctl_data_print(io); if (((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN) || @@ -12303,7 +12291,7 @@ ctl_handle_isc(union ctl_io *io) ctl_datamove_remote(io); break; case CTL_MSG_DATAMOVE_DONE: /* Only used in XFER mode */ - io->scsiio.be_move_done(io); + ctl_datamove_done(io, false); break; case CTL_MSG_FAILOVER: ctl_failover_lun(io); @@ -12462,6 +12450,45 @@ ctl_datamove_timer_wakeup(void *arg) } #endif /* CTL_IO_DELAY */ +static void +ctl_datamove_done_process(union ctl_io *io) +{ +#ifdef CTL_TIME_IO + struct bintime cur_bt; +#endif + + KASSERT(io->io_hdr.io_type == CTL_IO_SCSI, + ("%s: unexpected I/O type %x", __func__, io->io_hdr.io_type)); + +#ifdef CTL_TIME_IO + getbinuptime(&cur_bt); + bintime_sub(&cur_bt, &io->io_hdr.dma_start_bt); + bintime_add(&io->io_hdr.dma_bt, &cur_bt); +#endif + io->io_hdr.num_dmas++; + + if ((io->io_hdr.port_status != 0) && + ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE || + (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) { + ctl_set_internal_failure(&io->scsiio, /*sks_valid*/ 1, + /*retry_count*/ io->io_hdr.port_status); + } else if (io->scsiio.kern_data_resid != 0 && + (io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT && + ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE || + (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) { + ctl_set_invalid_field_ciu(&io->scsiio); + } else if (ctl_debug & CTL_DEBUG_CDB_DATA) + ctl_data_print(io); +} + +void +ctl_datamove_done(union ctl_io *io, bool samethr) +{ + + ctl_datamove_done_process(io); + io->scsiio.be_move_done(io, samethr); +} + void ctl_datamove(union ctl_io *io) { @@ -12475,39 +12502,7 @@ ctl_datamove(union ctl_io *io) io->scsiio.kern_data_resid = io->scsiio.kern_data_len; #ifdef CTL_TIME_IO - if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) { - char str[256]; - char path_str[64]; - struct sbuf sb; - - ctl_scsi_path_string(io, path_str, sizeof(path_str)); - sbuf_new(&sb, str, sizeof(str), SBUF_FIXEDLEN); - - sbuf_cat(&sb, path_str); - switch (io->io_hdr.io_type) { - case CTL_IO_SCSI: - ctl_scsi_command_string(&io->scsiio, NULL, &sb); - sbuf_printf(&sb, "\n"); - sbuf_cat(&sb, path_str); - sbuf_printf(&sb, "Tag: 0x%04x/%d, Prio: %d\n", - io->scsiio.tag_num, io->scsiio.tag_type, - io->scsiio.priority); - break; - case CTL_IO_TASK: - sbuf_printf(&sb, "Task Action: %d Tag: 0x%04x/%d\n", - io->taskio.task_action, - io->taskio.tag_num, io->taskio.tag_type); - break; - default: - panic("%s: Invalid CTL I/O type %d\n", - __func__, io->io_hdr.io_type); - } - sbuf_cat(&sb, path_str); - sbuf_printf(&sb, "ctl_datamove: %jd seconds\n", - (intmax_t)time_uptime - io->io_hdr.start_time); - sbuf_finish(&sb); - printf("%s", sbuf_data(&sb)); - } + getbinuptime(&io->io_hdr.dma_start_bt); #endif /* CTL_TIME_IO */ #ifdef CTL_IO_DELAY @@ -12542,18 +12537,15 @@ ctl_datamove(union ctl_io *io) io->io_hdr.nexus.targ_port, io->io_hdr.nexus.targ_lun); io->io_hdr.port_status = 31337; - /* - * Note that the backend, in this case, will get the - * callback in its context. In other cases it may get - * called in the frontend's interrupt thread context. - */ - io->scsiio.be_move_done(io); + ctl_datamove_done_process(io); + io->scsiio.be_move_done(io, true); return; } /* Don't confuse frontend with zero length data move. */ if (io->scsiio.kern_data_len == 0) { - io->scsiio.be_move_done(io); + ctl_datamove_done_process(io); + io->scsiio.be_move_done(io, true); return; } @@ -12640,7 +12632,7 @@ ctl_datamove_remote_write_cb(struct ctl_ha_dt_req *rq) * need to push it over to the remote controller's memory. */ static int -ctl_datamove_remote_dm_write_cb(union ctl_io *io) +ctl_datamove_remote_dm_write_cb(union ctl_io *io, bool samethr) { int retval; @@ -12679,7 +12671,7 @@ ctl_datamove_remote_write(union ctl_io *io) } static int -ctl_datamove_remote_dm_read_cb(union ctl_io *io) +ctl_datamove_remote_dm_read_cb(union ctl_io *io, bool samethr) { uint32_t i; diff --git a/sys/cam/ctl/ctl.h b/sys/cam/ctl/ctl.h index 56dd5313b4cb..be3e4a37b157 100644 --- a/sys/cam/ctl/ctl.h +++ b/sys/cam/ctl/ctl.h @@ -170,7 +170,8 @@ int ctl_sap_log_sense_handler(struct ctl_scsiio *ctsio, int ctl_ie_log_sense_handler(struct ctl_scsiio *ctsio, struct ctl_page_index *page_index, int pc); -int ctl_config_move_done(union ctl_io *io); +int ctl_config_move_done(union ctl_io *io, bool samethr); +void ctl_datamove_done(union ctl_io *io, bool samethr); void ctl_datamove(union ctl_io *io); void ctl_serseq_done(union ctl_io *io); void ctl_done(union ctl_io *io); diff --git a/sys/cam/ctl/ctl_backend.h b/sys/cam/ctl/ctl_backend.h index be8ab4d1706b..05e65abe41f8 100644 --- a/sys/cam/ctl/ctl_backend.h +++ b/sys/cam/ctl/ctl_backend.h @@ -186,7 +186,6 @@ struct ctl_backend_driver { be_init_t init; /* passed to CTL */ be_shutdown_t shutdown; /* passed to CTL */ be_func_t data_submit; /* passed to CTL */ - be_func_t data_move_done; /* passed to CTL */ be_func_t config_read; /* passed to CTL */ be_func_t config_write; /* passed to CTL */ be_ioctl_t ioctl; /* passed to CTL */ diff --git a/sys/cam/ctl/ctl_backend_block.c b/sys/cam/ctl/ctl_backend_block.c index 5262c041bf62..7ec8e32f7338 100644 --- a/sys/cam/ctl/ctl_backend_block.c +++ b/sys/cam/ctl/ctl_backend_block.c @@ -235,7 +235,7 @@ SYSCTL_INT(_kern_cam_ctl_block, OID_AUTO, num_threads, CTLFLAG_RWTUN, static struct ctl_be_block_io *ctl_alloc_beio(struct ctl_be_block_softc *softc); static void ctl_free_beio(struct ctl_be_block_io *beio); static void ctl_complete_beio(struct ctl_be_block_io *beio); -static int ctl_be_block_move_done(union ctl_io *io); +static int ctl_be_block_move_done(union ctl_io *io, bool samethr); static void ctl_be_block_biodone(struct bio *bio); static void ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun, struct ctl_be_block_io *beio); @@ -291,7 +291,6 @@ static struct ctl_backend_driver ctl_be_block_driver = .init = ctl_be_block_init, .shutdown = ctl_be_block_shutdown, .data_submit = ctl_be_block_submit, - .data_move_done = ctl_be_block_move_done, .config_read = ctl_be_block_config_read, .config_write = ctl_be_block_config_write, .ioctl = ctl_be_block_ioctl, @@ -432,46 +431,23 @@ ctl_be_block_compare(union ctl_io *io) } static int -ctl_be_block_move_done(union ctl_io *io) +ctl_be_block_move_done(union ctl_io *io, bool samethr) { struct ctl_be_block_io *beio; struct ctl_be_block_lun *be_lun; struct ctl_lba_len_flags *lbalen; -#ifdef CTL_TIME_IO - struct bintime cur_bt; -#endif beio = (struct ctl_be_block_io *)PRIV(io)->ptr; be_lun = beio->lun; DPRINTF("entered\n"); - -#ifdef CTL_TIME_IO - getbinuptime(&cur_bt); - bintime_sub(&cur_bt, &io->io_hdr.dma_start_bt); - bintime_add(&io->io_hdr.dma_bt, &cur_bt); -#endif - io->io_hdr.num_dmas++; io->scsiio.kern_rel_offset += io->scsiio.kern_data_len; /* - * We set status at this point for read commands, and write - * commands with errors. + * We set status at this point for read and compare commands. */ - if (io->io_hdr.flags & CTL_FLAG_ABORT) { - ; - } else if ((io->io_hdr.port_status != 0) && - ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE || - (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) { - ctl_set_internal_failure(&io->scsiio, /*sks_valid*/ 1, - /*retry_count*/ io->io_hdr.port_status); - } else if (io->scsiio.kern_data_resid != 0 && - (io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT && - ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE || - (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) { - ctl_set_invalid_field_ciu(&io->scsiio); - } else if ((io->io_hdr.port_status == 0) && - ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)) { + if ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0 && + (io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE) { lbalen = ARGS(beio->io); if (lbalen->flags & CTL_LLF_READ) { ctl_set_success(&io->scsiio); @@ -492,18 +468,22 @@ ctl_be_block_move_done(union ctl_io *io) } /* - * At this point, we have a write and the DMA completed - * successfully. We now have to queue it to the task queue to + * At this point, we have a write and the DMA completed successfully. + * If we were called synchronously in the original thread then just + * dispatch, otherwise we now have to queue it to the task queue to * execute the backend I/O. That is because we do blocking * memory allocations, and in the file backing case, blocking I/O. * This move done routine is generally called in the SIM's * interrupt context, and therefore we cannot block. */ - mtx_lock(&be_lun->queue_lock); - STAILQ_INSERT_TAIL(&be_lun->datamove_queue, &io->io_hdr, links); - mtx_unlock(&be_lun->queue_lock); - taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task); - + if (samethr) { + be_lun->dispatch(be_lun, beio); + } else { + mtx_lock(&be_lun->queue_lock); + STAILQ_INSERT_TAIL(&be_lun->datamove_queue, &io->io_hdr, links); + mtx_unlock(&be_lun->queue_lock); + taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task); + } return (0); } @@ -598,9 +578,6 @@ ctl_be_block_biodone(struct bio *bio) ctl_set_success(&io->scsiio); ctl_serseq_done(io); } -#ifdef CTL_TIME_IO - getbinuptime(&io->io_hdr.dma_start_bt); -#endif ctl_datamove(io); } } @@ -811,9 +788,6 @@ ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun, ctl_set_success(&io->scsiio); ctl_serseq_done(io); } -#ifdef CTL_TIME_IO - getbinuptime(&io->io_hdr.dma_start_bt); -#endif ctl_datamove(io); } } @@ -980,9 +954,6 @@ ctl_be_block_dispatch_zvol(struct ctl_be_block_lun *be_lun, ctl_set_success(&io->scsiio); ctl_serseq_done(io); } -#ifdef CTL_TIME_IO - getbinuptime(&io->io_hdr.dma_start_bt); -#endif ctl_datamove(io); } } @@ -1672,9 +1643,6 @@ ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun, be_lun->dispatch(be_lun, beio); } else { SDT_PROBE0(cbb, , write, alloc_done); -#ifdef CTL_TIME_IO - getbinuptime(&io->io_hdr.dma_start_bt); -#endif ctl_datamove(io); } } diff --git a/sys/cam/ctl/ctl_backend_ramdisk.c b/sys/cam/ctl/ctl_backend_ramdisk.c index 2595aa0be00e..e67d699bda70 100644 --- a/sys/cam/ctl/ctl_backend_ramdisk.c +++ b/sys/cam/ctl/ctl_backend_ramdisk.c @@ -139,7 +139,7 @@ extern struct ctl_softc *control_softc; static int ctl_backend_ramdisk_init(void); static int ctl_backend_ramdisk_shutdown(void); -static int ctl_backend_ramdisk_move_done(union ctl_io *io); +static int ctl_backend_ramdisk_move_done(union ctl_io *io, bool samethr); static void ctl_backend_ramdisk_compare(union ctl_io *io); static void ctl_backend_ramdisk_rw(union ctl_io *io); static int ctl_backend_ramdisk_submit(union ctl_io *io); @@ -164,7 +164,6 @@ static struct ctl_backend_driver ctl_be_ramdisk_driver = .init = ctl_backend_ramdisk_init, .shutdown = ctl_backend_ramdisk_shutdown, .data_submit = ctl_backend_ramdisk_submit, - .data_move_done = ctl_backend_ramdisk_move_done, .config_read = ctl_backend_ramdisk_config_read, .config_write = ctl_backend_ramdisk_config_write, .ioctl = ctl_backend_ramdisk_ioctl, @@ -402,38 +401,17 @@ ctl_backend_ramdisk_cmp(union ctl_io *io) } static int -ctl_backend_ramdisk_move_done(union ctl_io *io) +ctl_backend_ramdisk_move_done(union ctl_io *io, bool samethr) { struct ctl_be_ramdisk_lun *be_lun = (struct ctl_be_ramdisk_lun *)CTL_BACKEND_LUN(io); -#ifdef CTL_TIME_IO - struct bintime cur_bt; -#endif CTL_DEBUG_PRINT(("ctl_backend_ramdisk_move_done\n")); -#ifdef CTL_TIME_IO - getbinuptime(&cur_bt); - bintime_sub(&cur_bt, &io->io_hdr.dma_start_bt); - bintime_add(&io->io_hdr.dma_bt, &cur_bt); -#endif - io->io_hdr.num_dmas++; if (io->scsiio.kern_sg_entries > 0) free(io->scsiio.kern_data_ptr, M_RAMDISK); io->scsiio.kern_rel_offset += io->scsiio.kern_data_len; - if (io->io_hdr.flags & CTL_FLAG_ABORT) { - ; - } else if (io->io_hdr.port_status != 0 && - ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE || - (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) { - ctl_set_internal_failure(&io->scsiio, /*sks_valid*/ 1, - /*retry_count*/ io->io_hdr.port_status); - } else if (io->scsiio.kern_data_resid != 0 && - (io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT && - ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE || - (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) { - ctl_set_invalid_field_ciu(&io->scsiio); - } else if ((io->io_hdr.port_status == 0) && - ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)) { + if ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0 && + (io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE) { if (ARGS(io)->flags & CTL_LLF_COMPARE) { /* We have data block ready for comparison. */ if (ctl_backend_ramdisk_cmp(io)) @@ -471,9 +449,6 @@ ctl_backend_ramdisk_compare(union ctl_io *io) io->scsiio.kern_sg_entries = 0; io->io_hdr.flags |= CTL_FLAG_ALLOCATED; PRIV(io)->len += lbas; -#ifdef CTL_TIME_IO - getbinuptime(&io->io_hdr.dma_start_bt); -#endif ctl_datamove(io); } @@ -534,9 +509,6 @@ nospc: ctl_set_success(&io->scsiio); ctl_serseq_done(io); } -#ifdef CTL_TIME_IO - getbinuptime(&io->io_hdr.dma_start_bt); -#endif ctl_datamove(io); } diff --git a/sys/cam/ctl/ctl_frontend_cam_sim.c b/sys/cam/ctl/ctl_frontend_cam_sim.c index fdcccee2f569..0e61a80e452c 100644 --- a/sys/cam/ctl/ctl_frontend_cam_sim.c +++ b/sys/cam/ctl/ctl_frontend_cam_sim.c @@ -415,7 +415,7 @@ cfcs_datamove(union ctl_io *io) xpt_done(ccb); } - io->scsiio.be_move_done(io); + ctl_datamove_done(io, true); } static void diff --git a/sys/cam/ctl/ctl_frontend_ioctl.c b/sys/cam/ctl/ctl_frontend_ioctl.c index ef5e2bd22a86..f326100cb013 100644 --- a/sys/cam/ctl/ctl_frontend_ioctl.c +++ b/sys/cam/ctl/ctl_frontend_ioctl.c @@ -566,7 +566,7 @@ cfi_submit_wait(union ctl_io *io) * will immediately call back and wake us up, * probably using our own context. */ - io->scsiio.be_move_done(io); + ctl_datamove_done(io, false); break; case CTL_IOCTL_DONE: mtx_unlock(¶ms.ioctl_mtx); diff --git a/sys/cam/ctl/ctl_frontend_iscsi.c b/sys/cam/ctl/ctl_frontend_iscsi.c index 73483fb155cc..fdbc06150f93 100644 --- a/sys/cam/ctl/ctl_frontend_iscsi.c +++ b/sys/cam/ctl/ctl_frontend_iscsi.c @@ -933,7 +933,7 @@ cfiscsi_pdu_handle_data_out(struct icl_pdu *request) cfiscsi_data_wait_free(cs, cdw); io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG; if (done) - io->scsiio.be_move_done(io); + ctl_datamove_done(io, false); else cfiscsi_datamove_out(io); } @@ -1146,7 +1146,7 @@ cfiscsi_session_terminate_tasks(struct cfiscsi_session *cs) */ cdw->cdw_ctl_io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG; cdw->cdw_ctl_io->scsiio.io_hdr.port_status = 42; - cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io); + ctl_datamove_done(cdw->cdw_ctl_io, false); cfiscsi_data_wait_free(cs, cdw); CFISCSI_SESSION_LOCK(cs); } @@ -2487,7 +2487,7 @@ cfiscsi_datamove_in(union ctl_io *io) CFISCSI_SESSION_DEBUG(cs, "buffer_offset = %zd, " "already sent the expected len", buffer_offset); #endif - io->scsiio.be_move_done(io); + ctl_datamove_done(io, true); return; } @@ -2508,7 +2508,7 @@ cfiscsi_datamove_in(union ctl_io *io) CFISCSI_SESSION_WARN(cs, "failed to " "allocate memory; dropping connection"); ctl_set_busy(&io->scsiio); - io->scsiio.be_move_done(io); + ctl_datamove_done(io, true); cfiscsi_session_terminate(cs); return; } @@ -2566,7 +2566,7 @@ cfiscsi_datamove_in(union ctl_io *io) "allocate memory; dropping connection"); icl_pdu_free(response); ctl_set_busy(&io->scsiio); - io->scsiio.be_move_done(io); + ctl_datamove_done(io, true); cfiscsi_session_terminate(cs); return; } @@ -2656,7 +2656,7 @@ cfiscsi_datamove_in(union ctl_io *io) cfiscsi_pdu_queue_cb(response, cb); } - io->scsiio.be_move_done(io); + ctl_datamove_done(io, true); } static void @@ -2685,9 +2685,10 @@ cfiscsi_datamove_out(union ctl_io *io) */ expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length); if (io->scsiio.kern_rel_offset >= expected_len) { - io->scsiio.be_move_done(io); + ctl_datamove_done(io, true); return; } + datamove_len = MIN(io->scsiio.kern_data_len, expected_len - io->scsiio.kern_rel_offset); @@ -2703,7 +2704,7 @@ cfiscsi_datamove_out(union ctl_io *io) CFISCSI_SESSION_WARN(cs, "failed to " "allocate memory; dropping connection"); ctl_set_busy(&io->scsiio); - io->scsiio.be_move_done(io); + ctl_datamove_done(io, true); cfiscsi_session_terminate(cs); return; } @@ -2750,7 +2751,7 @@ cfiscsi_datamove_out(union ctl_io *io) done = cfiscsi_handle_data_segment(request, cdw); if (done) { cfiscsi_data_wait_free(cs, cdw); - io->scsiio.be_move_done(io); + ctl_datamove_done(io, true); return; } } @@ -2773,7 +2774,7 @@ cfiscsi_datamove_out(union ctl_io *io) CFISCSI_SESSION_WARN(cs, "failed to " "allocate memory; dropping connection"); ctl_set_busy(&io->scsiio); - io->scsiio.be_move_done(io); + ctl_datamove_done(io, true); cfiscsi_session_terminate(cs); return; } @@ -2954,7 +2955,7 @@ cfiscsi_task_management_done(union ctl_io *io) cdw, cdw_next); io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG; cdw->cdw_ctl_io->scsiio.io_hdr.port_status = 43; - cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io); + ctl_datamove_done(cdw->cdw_ctl_io, false); cfiscsi_data_wait_free(cs, cdw); } CFISCSI_SESSION_UNLOCK(cs); diff --git a/sys/cam/ctl/ctl_io.h b/sys/cam/ctl/ctl_io.h index 52ba98f3a9bd..60f8aef82d02 100644 --- a/sys/cam/ctl/ctl_io.h +++ b/sys/cam/ctl/ctl_io.h @@ -329,7 +329,7 @@ struct ctl_scsiio { ctl_tag_type tag_type; /* simple, ordered, head of queue,etc.*/ uint8_t cdb_len; /* CDB length */ uint8_t cdb[CTL_MAX_CDBLEN]; /* CDB */ - int (*be_move_done)(union ctl_io *io); /* called by fe */ + int (*be_move_done)(union ctl_io *io, bool samethr); /* called by fe */ int (*io_cont)(union ctl_io *io); /* to continue processing */ ctl_ref kern_data_ref; /* Method to reference/release data */ void *kern_data_arg; /* Opaque argument for kern_data_ref() */ diff --git a/sys/cam/ctl/ctl_tpc_local.c b/sys/cam/ctl/ctl_tpc_local.c index 714240fe8228..9b2c85ac8173 100644 --- a/sys/cam/ctl/ctl_tpc_local.c +++ b/sys/cam/ctl/ctl_tpc_local.c @@ -254,7 +254,7 @@ tpcl_datamove(union ctl_io *io) __func__, ctsio->ext_data_len, ctsio->kern_data_len)); bailout: - io->scsiio.be_move_done(io); + ctl_datamove_done(io, true); } static void diff --git a/sys/cam/ctl/scsi_ctl.c b/sys/cam/ctl/scsi_ctl.c index 646b3fe07053..d3023f9a6c8c 100644 --- a/sys/cam/ctl/scsi_ctl.c +++ b/sys/cam/ctl/scsi_ctl.c @@ -1394,8 +1394,7 @@ ctlfedone(struct cam_periph *periph, union ccb *done_ccb) xpt_release_ccb(done_ccb); mtx_unlock(mtx); - /* Call the backend move done callback */ - io->scsiio.be_move_done(io); + ctl_datamove_done(io, false); } return; } diff --git a/sys/dev/usb/storage/cfumass.c b/sys/dev/usb/storage/cfumass.c index 59d744bd62d0..88b5a6156704 100644 --- a/sys/dev/usb/storage/cfumass.c +++ b/sys/dev/usb/storage/cfumass.c @@ -475,7 +475,7 @@ cfumass_terminate(struct cfumass_softc *sc) if (sc->sc_ctl_io != NULL) { CFUMASS_DEBUG(sc, "terminating CTL transfer"); ctl_set_data_phase_error(&sc->sc_ctl_io->scsiio); - sc->sc_ctl_io->scsiio.be_move_done(sc->sc_ctl_io); + ctl_datamove_done(sc->sc_ctl_io, false); sc->sc_ctl_io = NULL; } @@ -730,7 +730,7 @@ cfumass_t_data_callback(struct usb_xfer *xfer, usb_error_t usb_error) sc->sc_current_residue == 0 || io->scsiio.kern_data_resid == 0) { sc->sc_ctl_io = NULL; - io->scsiio.be_move_done(io); + ctl_datamove_done(io, false); break; } /* FALLTHROUGH */ @@ -887,7 +887,7 @@ cfumass_datamove(union ctl_io *io) fail: ctl_set_data_phase_error(&io->scsiio); - io->scsiio.be_move_done(io); + ctl_datamove_done(io, true); sc->sc_ctl_io = NULL; }