From owner-svn-src-all@FreeBSD.ORG Thu Dec 18 08:22:20 2014 Return-Path: Delivered-To: svn-src-all@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by hub.freebsd.org (Postfix) with ESMTPS id 528EBAFC; Thu, 18 Dec 2014 08:22:20 +0000 (UTC) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 3CFB21754; Thu, 18 Dec 2014 08:22:20 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.9/8.14.9) with ESMTP id sBI8MKpn019415; Thu, 18 Dec 2014 08:22:20 GMT (envelope-from mav@FreeBSD.org) Received: (from mav@localhost) by svn.freebsd.org (8.14.9/8.14.9/Submit) id sBI8MHv8019400; Thu, 18 Dec 2014 08:22:17 GMT (envelope-from mav@FreeBSD.org) Message-Id: <201412180822.sBI8MHv8019400@svn.freebsd.org> X-Authentication-Warning: svn.freebsd.org: mav set sender to mav@FreeBSD.org using -f From: Alexander Motin Date: Thu, 18 Dec 2014 08:22:17 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-10@freebsd.org Subject: svn commit: r275878 - stable/10/sys/cam/ctl X-SVN-Group: stable-10 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.18-1 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 18 Dec 2014 08:22:20 -0000 Author: mav Date: Thu Dec 18 08:22:16 2014 New Revision: 275878 URL: https://svnweb.freebsd.org/changeset/base/275878 Log: MFC r274962: Replace home-grown CTL IO allocator with UMA. Old allocator created significant lock congestion protecting its lists of preallocated I/Os, while UMA provides much better SMP scalability. The downside of UMA is lack of reliable preallocation, that could guarantee successful allocation in non-sleepable environments. But careful code review shown, that only CAM target frontend really has that requirement. Fix that making that frontend preallocate and statically bind CTL I/O for every ATIO/INOT it preallocates any way. That allows to avoid allocations in hot I/O path. Other frontends either may sleep in allocation context or can properly handle allocation errors. On 40-core server with 6 ZVOL-backed LUNs and 7 iSCSI client connections this change increases peak performance from ~700K to >1M IOPS! Yay! :) Sponsored by: iXsystems, Inc. Modified: stable/10/sys/cam/ctl/ctl.c stable/10/sys/cam/ctl/ctl_frontend.c stable/10/sys/cam/ctl/ctl_frontend_cam_sim.c stable/10/sys/cam/ctl/ctl_frontend_internal.c stable/10/sys/cam/ctl/ctl_frontend_iscsi.c stable/10/sys/cam/ctl/ctl_io.h stable/10/sys/cam/ctl/ctl_private.h stable/10/sys/cam/ctl/ctl_tpc.c stable/10/sys/cam/ctl/scsi_ctl.c Directory Properties: stable/10/ (props changed) Modified: stable/10/sys/cam/ctl/ctl.c ============================================================================== --- stable/10/sys/cam/ctl/ctl.c Thu Dec 18 07:38:41 2014 (r275877) +++ stable/10/sys/cam/ctl/ctl.c Thu Dec 18 08:22:16 2014 (r275878) @@ -64,6 +64,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include @@ -646,7 +647,7 @@ ctl_isc_event_handler(ctl_ha_channel cha #if 0 printf("Serialize\n"); #endif - io = ctl_alloc_io((void *)ctl_softc->othersc_pool); + io = ctl_alloc_io_nowait(ctl_softc->othersc_pool); if (io == NULL) { printf("ctl_isc_event_handler: can't allocate " "ctl_io!\n"); @@ -891,8 +892,8 @@ ctl_isc_event_handler(ctl_ha_channel cha /* Handle resets sent from the other side */ case CTL_MSG_MANAGE_TASKS: { struct ctl_taskio *taskio; - taskio = (struct ctl_taskio *)ctl_alloc_io( - (void *)ctl_softc->othersc_pool); + taskio = (struct ctl_taskio *)ctl_alloc_io_nowait( + ctl_softc->othersc_pool); if (taskio == NULL) { printf("ctl_isc_event_handler: can't allocate " "ctl_io!\n"); @@ -920,8 +921,8 @@ ctl_isc_event_handler(ctl_ha_channel cha } /* Persistent Reserve action which needs attention */ case CTL_MSG_PERS_ACTION: - presio = (struct ctl_prio *)ctl_alloc_io( - (void *)ctl_softc->othersc_pool); + presio = (struct ctl_prio *)ctl_alloc_io_nowait( + ctl_softc->othersc_pool); if (presio == NULL) { printf("ctl_isc_event_handler: can't allocate " "ctl_io!\n"); @@ -1005,7 +1006,7 @@ static int ctl_init(void) { struct ctl_softc *softc; - struct ctl_io_pool *internal_pool, *emergency_pool, *other_pool; + void *other_pool; struct ctl_port *port; int i, error, retval; //int isc_retval; @@ -1051,7 +1052,8 @@ ctl_init(void) "Report no lun possible for invalid LUNs"); mtx_init(&softc->ctl_lock, "CTL mutex", NULL, MTX_DEF); - mtx_init(&softc->pool_lock, "CTL pool mutex", NULL, MTX_DEF); + softc->io_zone = uma_zcreate("CTL IO", sizeof(union ctl_io), + NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); softc->open_count = 0; /* @@ -1088,36 +1090,15 @@ ctl_init(void) STAILQ_INIT(&softc->fe_list); STAILQ_INIT(&softc->port_list); STAILQ_INIT(&softc->be_list); - STAILQ_INIT(&softc->io_pools); ctl_tpc_init(softc); - if (ctl_pool_create(softc, CTL_POOL_INTERNAL, CTL_POOL_ENTRIES_INTERNAL, - &internal_pool)!= 0){ - printf("ctl: can't allocate %d entry internal pool, " - "exiting\n", CTL_POOL_ENTRIES_INTERNAL); - return (ENOMEM); - } - - if (ctl_pool_create(softc, CTL_POOL_EMERGENCY, - CTL_POOL_ENTRIES_EMERGENCY, &emergency_pool) != 0) { - printf("ctl: can't allocate %d entry emergency pool, " - "exiting\n", CTL_POOL_ENTRIES_EMERGENCY); - ctl_pool_free(internal_pool); - return (ENOMEM); - } - - if (ctl_pool_create(softc, CTL_POOL_4OTHERSC, CTL_POOL_ENTRIES_OTHER_SC, + if (ctl_pool_create(softc, "othersc", CTL_POOL_ENTRIES_OTHER_SC, &other_pool) != 0) { printf("ctl: can't allocate %d entry other SC pool, " "exiting\n", CTL_POOL_ENTRIES_OTHER_SC); - ctl_pool_free(internal_pool); - ctl_pool_free(emergency_pool); return (ENOMEM); } - - softc->internal_pool = internal_pool; - softc->emergency_pool = emergency_pool; softc->othersc_pool = other_pool; if (worker_threads <= 0) @@ -1139,8 +1120,6 @@ ctl_init(void) &softc->ctl_proc, &thr->thread, 0, 0, "ctl", "work%d", i); if (error != 0) { printf("error creating CTL work thread!\n"); - ctl_pool_free(internal_pool); - ctl_pool_free(emergency_pool); ctl_pool_free(other_pool); return (error); } @@ -1149,8 +1128,6 @@ ctl_init(void) &softc->ctl_proc, NULL, 0, 0, "ctl", "lun"); if (error != 0) { printf("error creating CTL lun thread!\n"); - ctl_pool_free(internal_pool); - ctl_pool_free(emergency_pool); ctl_pool_free(other_pool); return (error); } @@ -1158,8 +1135,6 @@ ctl_init(void) &softc->ctl_proc, NULL, 0, 0, "ctl", "thresh"); if (error != 0) { printf("error creating CTL threshold thread!\n"); - ctl_pool_free(internal_pool); - ctl_pool_free(emergency_pool); ctl_pool_free(other_pool); return (error); } @@ -1212,7 +1187,6 @@ ctl_shutdown(void) { struct ctl_softc *softc; struct ctl_lun *lun, *next_lun; - struct ctl_io_pool *pool; softc = (struct ctl_softc *)control_softc; @@ -1233,24 +1207,13 @@ ctl_shutdown(void) ctl_frontend_deregister(&ioctl_frontend); - /* - * This will rip the rug out from under any FETDs or anyone else - * that has a pool allocated. Since we increment our module - * refcount any time someone outside the main CTL module allocates - * a pool, we shouldn't have any problems here. The user won't be - * able to unload the CTL module until client modules have - * successfully unloaded. - */ - while ((pool = STAILQ_FIRST(&softc->io_pools)) != NULL) - ctl_pool_free(pool); - #if 0 ctl_shutdown_thread(softc->work_thread); mtx_destroy(&softc->queue_lock); #endif ctl_tpc_shutdown(softc); - mtx_destroy(&softc->pool_lock); + uma_zdestroy(softc->io_zone); mtx_destroy(&softc->ctl_lock); destroy_dev(softc->dev); @@ -2373,21 +2336,15 @@ ctl_ioctl(struct cdev *dev, u_long cmd, } io = ctl_alloc_io(softc->ioctl_info.port.ctl_pool_ref); - if (io == NULL) { - printf("ctl_ioctl: can't allocate ctl_io!\n"); - retval = ENOSPC; - break; - } /* * Need to save the pool reference so it doesn't get * spammed by the user's ctl_io. */ pool_tmp = io->io_hdr.pool; - memcpy(io, (void *)addr, sizeof(*io)); - io->io_hdr.pool = pool_tmp; + /* * No status yet, so make sure the status is set properly. */ @@ -3731,285 +3688,95 @@ ctl_kfree_io(union ctl_io *io) #endif /* unused */ /* - * ctl_softc, pool_type, total_ctl_io are passed in. + * ctl_softc, pool_name, total_ctl_io are passed in. * npool is passed out. */ int -ctl_pool_create(struct ctl_softc *ctl_softc, ctl_pool_type pool_type, - uint32_t total_ctl_io, struct ctl_io_pool **npool) +ctl_pool_create(struct ctl_softc *ctl_softc, const char *pool_name, + uint32_t total_ctl_io, void **npool) { - uint32_t i; - union ctl_io *cur_io, *next_io; +#ifdef IO_POOLS struct ctl_io_pool *pool; - int retval; - - retval = 0; pool = (struct ctl_io_pool *)malloc(sizeof(*pool), M_CTL, M_NOWAIT | M_ZERO); - if (pool == NULL) { - retval = ENOMEM; - goto bailout; - } + if (pool == NULL) + return (ENOMEM); - pool->type = pool_type; + snprintf(pool->name, sizeof(pool->name), "CTL IO %s", pool_name); pool->ctl_softc = ctl_softc; - - mtx_lock(&ctl_softc->pool_lock); - pool->id = ctl_softc->cur_pool_id++; - mtx_unlock(&ctl_softc->pool_lock); - - pool->flags = CTL_POOL_FLAG_NONE; - pool->refcount = 1; /* Reference for validity. */ - STAILQ_INIT(&pool->free_queue); - - /* - * XXX KDM other options here: - * - allocate a page at a time - * - allocate one big chunk of memory. - * Page allocation might work well, but would take a little more - * tracking. - */ - for (i = 0; i < total_ctl_io; i++) { - cur_io = (union ctl_io *)malloc(sizeof(*cur_io), M_CTLIO, - M_NOWAIT); - if (cur_io == NULL) { - retval = ENOMEM; - break; - } - cur_io->io_hdr.pool = pool; - STAILQ_INSERT_TAIL(&pool->free_queue, &cur_io->io_hdr, links); - pool->total_ctl_io++; - pool->free_ctl_io++; - } - - if (retval != 0) { - for (cur_io = (union ctl_io *)STAILQ_FIRST(&pool->free_queue); - cur_io != NULL; cur_io = next_io) { - next_io = (union ctl_io *)STAILQ_NEXT(&cur_io->io_hdr, - links); - STAILQ_REMOVE(&pool->free_queue, &cur_io->io_hdr, - ctl_io_hdr, links); - free(cur_io, M_CTLIO); - } - - free(pool, M_CTL); - goto bailout; - } - mtx_lock(&ctl_softc->pool_lock); - ctl_softc->num_pools++; - STAILQ_INSERT_TAIL(&ctl_softc->io_pools, pool, links); - /* - * Increment our usage count if this is an external consumer, so we - * can't get unloaded until the external consumer (most likely a - * FETD) unloads and frees his pool. - * - * XXX KDM will this increment the caller's module use count, or - * mine? - */ -#if 0 - if ((pool_type != CTL_POOL_EMERGENCY) - && (pool_type != CTL_POOL_INTERNAL) - && (pool_type != CTL_POOL_4OTHERSC)) - MOD_INC_USE_COUNT; -#endif - - mtx_unlock(&ctl_softc->pool_lock); + pool->zone = uma_zsecond_create(pool->name, NULL, + NULL, NULL, NULL, ctl_softc->io_zone); + /* uma_prealloc(pool->zone, total_ctl_io); */ *npool = pool; - -bailout: - - return (retval); -} - -static int -ctl_pool_acquire(struct ctl_io_pool *pool) -{ - - mtx_assert(&pool->ctl_softc->pool_lock, MA_OWNED); - - if (pool->flags & CTL_POOL_FLAG_INVALID) - return (EINVAL); - - pool->refcount++; - - return (0); -} - -static void -ctl_pool_release(struct ctl_io_pool *pool) -{ - struct ctl_softc *ctl_softc = pool->ctl_softc; - union ctl_io *io; - - mtx_assert(&ctl_softc->pool_lock, MA_OWNED); - - if (--pool->refcount != 0) - return; - - while ((io = (union ctl_io *)STAILQ_FIRST(&pool->free_queue)) != NULL) { - STAILQ_REMOVE(&pool->free_queue, &io->io_hdr, ctl_io_hdr, - links); - free(io, M_CTLIO); - } - - STAILQ_REMOVE(&ctl_softc->io_pools, pool, ctl_io_pool, links); - ctl_softc->num_pools--; - - /* - * XXX KDM will this decrement the caller's usage count or mine? - */ -#if 0 - if ((pool->type != CTL_POOL_EMERGENCY) - && (pool->type != CTL_POOL_INTERNAL) - && (pool->type != CTL_POOL_4OTHERSC)) - MOD_DEC_USE_COUNT; +#else + *npool = ctl_softc->io_zone; #endif - - free(pool, M_CTL); + return (0); } void ctl_pool_free(struct ctl_io_pool *pool) { - struct ctl_softc *ctl_softc; if (pool == NULL) return; - ctl_softc = pool->ctl_softc; - mtx_lock(&ctl_softc->pool_lock); - pool->flags |= CTL_POOL_FLAG_INVALID; - ctl_pool_release(pool); - mtx_unlock(&ctl_softc->pool_lock); +#ifdef IO_POOLS + uma_zdestroy(pool->zone); + free(pool, M_CTL); +#endif } -/* - * This routine does not block (except for spinlocks of course). - * It tries to allocate a ctl_io union from the caller's pool as quickly as - * possible. - */ union ctl_io * ctl_alloc_io(void *pool_ref) { union ctl_io *io; - struct ctl_softc *ctl_softc; - struct ctl_io_pool *pool, *npool; - struct ctl_io_pool *emergency_pool; +#ifdef IO_POOLS + struct ctl_io_pool *pool = (struct ctl_io_pool *)pool_ref; - pool = (struct ctl_io_pool *)pool_ref; - - if (pool == NULL) { - printf("%s: pool is NULL\n", __func__); - return (NULL); - } - - emergency_pool = NULL; - - ctl_softc = pool->ctl_softc; - - mtx_lock(&ctl_softc->pool_lock); - /* - * First, try to get the io structure from the user's pool. - */ - if (ctl_pool_acquire(pool) == 0) { - io = (union ctl_io *)STAILQ_FIRST(&pool->free_queue); - if (io != NULL) { - STAILQ_REMOVE_HEAD(&pool->free_queue, links); - pool->total_allocated++; - pool->free_ctl_io--; - mtx_unlock(&ctl_softc->pool_lock); - return (io); - } else - ctl_pool_release(pool); - } - /* - * If he doesn't have any io structures left, search for an - * emergency pool and grab one from there. - */ - STAILQ_FOREACH(npool, &ctl_softc->io_pools, links) { - if (npool->type != CTL_POOL_EMERGENCY) - continue; - - if (ctl_pool_acquire(npool) != 0) - continue; - - emergency_pool = npool; - - io = (union ctl_io *)STAILQ_FIRST(&npool->free_queue); - if (io != NULL) { - STAILQ_REMOVE_HEAD(&npool->free_queue, links); - npool->total_allocated++; - npool->free_ctl_io--; - mtx_unlock(&ctl_softc->pool_lock); - return (io); - } else - ctl_pool_release(npool); - } - - /* Drop the spinlock before we malloc */ - mtx_unlock(&ctl_softc->pool_lock); + io = uma_zalloc(pool->zone, M_WAITOK); +#else + io = uma_zalloc((uma_zone_t)pool_ref, M_WAITOK); +#endif + if (io != NULL) + io->io_hdr.pool = pool_ref; + return (io); +} - /* - * The emergency pool (if it exists) didn't have one, so try an - * atomic (i.e. nonblocking) malloc and see if we get lucky. - */ - io = (union ctl_io *)malloc(sizeof(*io), M_CTLIO, M_NOWAIT); - if (io != NULL) { - /* - * If the emergency pool exists but is empty, add this - * ctl_io to its list when it gets freed. - */ - if (emergency_pool != NULL) { - mtx_lock(&ctl_softc->pool_lock); - if (ctl_pool_acquire(emergency_pool) == 0) { - io->io_hdr.pool = emergency_pool; - emergency_pool->total_ctl_io++; - /* - * Need to bump this, otherwise - * total_allocated and total_freed won't - * match when we no longer have anything - * outstanding. - */ - emergency_pool->total_allocated++; - } - mtx_unlock(&ctl_softc->pool_lock); - } else - io->io_hdr.pool = NULL; - } +union ctl_io * +ctl_alloc_io_nowait(void *pool_ref) +{ + union ctl_io *io; +#ifdef IO_POOLS + struct ctl_io_pool *pool = (struct ctl_io_pool *)pool_ref; + io = uma_zalloc(pool->zone, M_NOWAIT); +#else + io = uma_zalloc((uma_zone_t)pool_ref, M_NOWAIT); +#endif + if (io != NULL) + io->io_hdr.pool = pool_ref; return (io); } void ctl_free_io(union ctl_io *io) { +#ifdef IO_POOLS + struct ctl_io_pool *pool; +#endif + if (io == NULL) return; - /* - * If this ctl_io has a pool, return it to that pool. - */ - if (io->io_hdr.pool != NULL) { - struct ctl_io_pool *pool; - - pool = (struct ctl_io_pool *)io->io_hdr.pool; - mtx_lock(&pool->ctl_softc->pool_lock); - io->io_hdr.io_type = 0xff; - STAILQ_INSERT_TAIL(&pool->free_queue, &io->io_hdr, links); - pool->total_freed++; - pool->free_ctl_io++; - ctl_pool_release(pool); - mtx_unlock(&pool->ctl_softc->pool_lock); - } else { - /* - * Otherwise, just free it. We probably malloced it and - * the emergency pool wasn't available. - */ - free(io, M_CTLIO); - } - +#ifdef IO_POOLS + pool = (struct ctl_io_pool *)io->io_hdr.pool; + uma_zfree(pool->zone, io); +#else + uma_zfree((uma_zone_t)io->io_hdr.pool, io); +#endif } void @@ -4024,9 +3791,7 @@ ctl_zero_io(union ctl_io *io) * May need to preserve linked list pointers at some point too. */ pool_ref = io->io_hdr.pool; - memset(io, 0, sizeof(*io)); - io->io_hdr.pool = pool_ref; } @@ -5659,16 +5424,10 @@ ctl_start_stop(struct ctl_scsiio *ctsio) union ctl_io *new_io; new_io = ctl_alloc_io(ctsio->io_hdr.pool); - if (new_io == NULL) { - ctl_set_busy(ctsio); - ctl_done((union ctl_io *)ctsio); - } else { - ctl_copy_io((union ctl_io *)ctsio, - new_io); - retval = lun->backend->config_write(new_io); - ctl_set_success(ctsio); - ctl_done((union ctl_io *)ctsio); - } + ctl_copy_io((union ctl_io *)ctsio, new_io); + retval = lun->backend->config_write(new_io); + ctl_set_success(ctsio); + ctl_done((union ctl_io *)ctsio); } else { retval = lun->backend->config_write( (union ctl_io *)ctsio); Modified: stable/10/sys/cam/ctl/ctl_frontend.c ============================================================================== --- stable/10/sys/cam/ctl/ctl_frontend.c Thu Dec 18 07:38:41 2014 (r275877) +++ stable/10/sys/cam/ctl/ctl_frontend.c Thu Dec 18 08:22:16 2014 (r275878) @@ -138,7 +138,7 @@ ctl_frontend_find(char *frontend_name) int ctl_port_register(struct ctl_port *port) { - struct ctl_io_pool *pool; + void *pool; int port_num; int retval; @@ -176,7 +176,7 @@ ctl_port_register(struct ctl_port *port) * pending sense queue on the next command, whether or not it is * a REQUEST SENSE. */ - retval = ctl_pool_create(control_softc, CTL_POOL_FETD, + retval = ctl_pool_create(control_softc, port->port_name, port->num_requested_ctl_io + 20, &pool); if (retval != 0) { free(port->wwpn_iid, M_CTL); Modified: stable/10/sys/cam/ctl/ctl_frontend_cam_sim.c ============================================================================== --- stable/10/sys/cam/ctl/ctl_frontend_cam_sim.c Thu Dec 18 07:38:41 2014 (r275877) +++ stable/10/sys/cam/ctl/ctl_frontend_cam_sim.c Thu Dec 18 08:22:16 2014 (r275878) @@ -545,7 +545,7 @@ cfcs_action(struct cam_sim *sim, union c return; } - io = ctl_alloc_io(softc->port.ctl_pool_ref); + io = ctl_alloc_io_nowait(softc->port.ctl_pool_ref); if (io == NULL) { printf("%s: can't allocate ctl_io\n", __func__); ccb->ccb_h.status = CAM_BUSY | CAM_DEV_QFRZN; @@ -642,7 +642,7 @@ cfcs_action(struct cam_sim *sim, union c return; } - io = ctl_alloc_io(softc->port.ctl_pool_ref); + io = ctl_alloc_io_nowait(softc->port.ctl_pool_ref); if (io == NULL) { ccb->ccb_h.status = CAM_BUSY | CAM_DEV_QFRZN; xpt_freeze_devq(ccb->ccb_h.path, 1); @@ -737,7 +737,7 @@ cfcs_action(struct cam_sim *sim, union c return; } - io = ctl_alloc_io(softc->port.ctl_pool_ref); + io = ctl_alloc_io_nowait(softc->port.ctl_pool_ref); if (io == NULL) { ccb->ccb_h.status = CAM_BUSY | CAM_DEV_QFRZN; xpt_freeze_devq(ccb->ccb_h.path, 1); Modified: stable/10/sys/cam/ctl/ctl_frontend_internal.c ============================================================================== --- stable/10/sys/cam/ctl/ctl_frontend_internal.c Thu Dec 18 07:38:41 2014 (r275877) +++ stable/10/sys/cam/ctl/ctl_frontend_internal.c Thu Dec 18 08:22:16 2014 (r275878) @@ -761,11 +761,6 @@ cfi_done(union ctl_io *io) struct cfi_lun_io *new_lun_io; new_io = ctl_alloc_io(softc->port.ctl_pool_ref); - if (new_io == NULL) { - printf("%s: unable to allocate ctl_io for " - "error recovery\n", __func__); - goto done; - } ctl_zero_io(new_io); new_io->io_hdr.io_type = CTL_IO_TASK; @@ -967,12 +962,6 @@ cfi_lun_probe(struct cfi_lun *lun, int h union ctl_io *io; io = ctl_alloc_io(lun->softc->port.ctl_pool_ref); - if (io == NULL) { - printf("%s: unable to alloc ctl_io for target %ju " - "lun %d probe\n", __func__, - (uintmax_t)lun->target_id.id, lun->lun_id); - return; - } ctl_scsi_inquiry(io, /*data_ptr*/(uint8_t *)&lun->inq_data, /*data_len*/ sizeof(lun->inq_data), @@ -1014,12 +1003,6 @@ cfi_lun_probe(struct cfi_lun *lun, int h union ctl_io *io; io = ctl_alloc_io(lun->softc->port.ctl_pool_ref); - if (io == NULL) { - printf("%s: unable to alloc ctl_io for target %ju " - "lun %d probe\n", __func__, - (uintmax_t)lun->target_id.id, lun->lun_id); - return; - } dataptr = malloc(sizeof(struct scsi_read_capacity_data_long), M_CTL_CFI, M_NOWAIT); @@ -1394,7 +1377,7 @@ cfi_action(struct cfi_metatask *metatask if (SID_TYPE(&lun->inq_data) != T_DIRECT) continue; da_luns++; - io = ctl_alloc_io(softc->port.ctl_pool_ref); + io = ctl_alloc_io_nowait(softc->port.ctl_pool_ref); if (io != NULL) { ios_allocated++; STAILQ_INSERT_TAIL(&tmp_io_list, &io->io_hdr, @@ -1548,7 +1531,7 @@ cfi_action(struct cfi_metatask *metatask } - io = ctl_alloc_io(softc->port.ctl_pool_ref); + io = ctl_alloc_io_nowait(softc->port.ctl_pool_ref); if (io == NULL) { metatask->status = CFI_MT_ERROR; metatask->taskinfo.bbrread.status = CFI_BBR_NO_MEM; Modified: stable/10/sys/cam/ctl/ctl_frontend_iscsi.c ============================================================================== --- stable/10/sys/cam/ctl/ctl_frontend_iscsi.c Thu Dec 18 07:38:41 2014 (r275877) +++ stable/10/sys/cam/ctl/ctl_frontend_iscsi.c Thu Dec 18 08:22:16 2014 (r275878) @@ -546,13 +546,6 @@ cfiscsi_pdu_handle_scsi_command(struct i return; } io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref); - if (io == NULL) { - CFISCSI_SESSION_WARN(cs, "can't allocate ctl_io; " - "dropping connection"); - icl_pdu_free(request); - cfiscsi_session_terminate(cs); - return; - } ctl_zero_io(io); io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = request; io->io_hdr.io_type = CTL_IO_SCSI; @@ -610,13 +603,6 @@ cfiscsi_pdu_handle_task_request(struct i cs = PDU_SESSION(request); bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs; io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref); - if (io == NULL) { - CFISCSI_SESSION_WARN(cs, "can't allocate ctl_io;" - "dropping connection"); - icl_pdu_free(request); - cfiscsi_session_terminate(cs); - return; - } ctl_zero_io(io); io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = request; io->io_hdr.io_type = CTL_IO_TASK; @@ -1067,10 +1053,6 @@ cfiscsi_session_terminate_tasks(struct c if (cs->cs_target == NULL) return; /* No target yet, so nothing to do. */ io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref); - if (io == NULL) { - CFISCSI_SESSION_WARN(cs, "can't allocate ctl_io"); - return; - } ctl_zero_io(io); io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = cs; io->io_hdr.io_type = CTL_IO_TASK; Modified: stable/10/sys/cam/ctl/ctl_io.h ============================================================================== --- stable/10/sys/cam/ctl/ctl_io.h Thu Dec 18 07:38:41 2014 (r275877) +++ stable/10/sys/cam/ctl/ctl_io.h Thu Dec 18 08:22:16 2014 (r275878) @@ -511,6 +511,7 @@ union ctl_io { #ifdef _KERNEL union ctl_io *ctl_alloc_io(void *pool_ref); +union ctl_io *ctl_alloc_io_nowait(void *pool_ref); void ctl_free_io(union ctl_io *io); void ctl_zero_io(union ctl_io *io); void ctl_copy_io(union ctl_io *src, union ctl_io *dest); Modified: stable/10/sys/cam/ctl/ctl_private.h ============================================================================== --- stable/10/sys/cam/ctl/ctl_private.h Thu Dec 18 07:38:41 2014 (r275877) +++ stable/10/sys/cam/ctl/ctl_private.h Thu Dec 18 08:22:16 2014 (r275878) @@ -71,34 +71,13 @@ struct ctl_fe_ioctl_params { ctl_fe_ioctl_state state; }; -#define CTL_POOL_ENTRIES_INTERNAL 200 -#define CTL_POOL_ENTRIES_EMERGENCY 300 #define CTL_POOL_ENTRIES_OTHER_SC 200 -typedef enum { - CTL_POOL_INTERNAL, - CTL_POOL_FETD, - CTL_POOL_EMERGENCY, - CTL_POOL_4OTHERSC -} ctl_pool_type; - -typedef enum { - CTL_POOL_FLAG_NONE = 0x00, - CTL_POOL_FLAG_INVALID = 0x01 -} ctl_pool_flags; - struct ctl_io_pool { - ctl_pool_type type; - ctl_pool_flags flags; + char name[64]; uint32_t id; struct ctl_softc *ctl_softc; - uint32_t refcount; - uint64_t total_allocated; - uint64_t total_freed; - int32_t total_ctl_io; - int32_t free_ctl_io; - STAILQ_HEAD(, ctl_io_hdr) free_queue; - STAILQ_ENTRY(ctl_io_pool) links; + struct uma_zone *zone; }; typedef enum { @@ -475,9 +454,7 @@ struct ctl_softc { struct sysctl_ctx_list sysctl_ctx; struct sysctl_oid *sysctl_tree; struct ctl_ioctl_info ioctl_info; - struct ctl_io_pool *internal_pool; - struct ctl_io_pool *emergency_pool; - struct ctl_io_pool *othersc_pool; + void *othersc_pool; struct proc *ctl_proc; int targ_online; uint32_t ctl_lun_mask[(CTL_MAX_LUNS + 31) / 32]; @@ -492,10 +469,8 @@ struct ctl_softc { struct ctl_port *ctl_ports[CTL_MAX_PORTS]; uint32_t num_backends; STAILQ_HEAD(, ctl_backend_driver) be_list; - struct mtx pool_lock; - uint32_t num_pools; + struct uma_zone *io_zone; uint32_t cur_pool_id; - STAILQ_HEAD(, ctl_io_pool) io_pools; struct ctl_thread threads[CTL_MAX_THREADS]; TAILQ_HEAD(tpc_tokens, tpc_token) tpc_tokens; struct callout tpc_timeout; @@ -508,8 +483,8 @@ extern const struct ctl_cmd_entry ctl_cm uint32_t ctl_get_initindex(struct ctl_nexus *nexus); uint32_t ctl_get_resindex(struct ctl_nexus *nexus); uint32_t ctl_port_idx(int port_num); -int ctl_pool_create(struct ctl_softc *ctl_softc, ctl_pool_type pool_type, - uint32_t total_ctl_io, struct ctl_io_pool **npool); +int ctl_pool_create(struct ctl_softc *ctl_softc, const char *pool_name, + uint32_t total_ctl_io, void **npool); void ctl_pool_free(struct ctl_io_pool *pool); int ctl_scsi_release(struct ctl_scsiio *ctsio); int ctl_scsi_reserve(struct ctl_scsiio *ctsio); Modified: stable/10/sys/cam/ctl/ctl_tpc.c ============================================================================== --- stable/10/sys/cam/ctl/ctl_tpc.c Thu Dec 18 07:38:41 2014 (r275877) +++ stable/10/sys/cam/ctl/ctl_tpc.c Thu Dec 18 08:22:16 2014 (r275878) @@ -812,7 +812,6 @@ tpc_process_b2b(struct tpc_list *list) uint32_t srcblock, dstblock; if (list->stage == 1) { -complete: while ((tior = TAILQ_FIRST(&list->allio)) != NULL) { TAILQ_REMOVE(&list->allio, tior, links); ctl_free_io(tior->io); @@ -886,10 +885,6 @@ complete: tior->list = list; TAILQ_INSERT_TAIL(&list->allio, tior, links); tior->io = tpcl_alloc_io(); - if (tior->io == NULL) { - list->error = 1; - goto complete; - } ctl_scsi_read_write(tior->io, /*data_ptr*/ &list->buf[donebytes], /*data_len*/ roundbytes, @@ -909,10 +904,6 @@ complete: tiow->list = list; TAILQ_INSERT_TAIL(&list->allio, tiow, links); tiow->io = tpcl_alloc_io(); - if (tiow->io == NULL) { - list->error = 1; - goto complete; - } ctl_scsi_read_write(tiow->io, /*data_ptr*/ &list->buf[donebytes], /*data_len*/ roundbytes, @@ -951,7 +942,6 @@ tpc_process_verify(struct tpc_list *list uint64_t sl; if (list->stage == 1) { -complete: while ((tio = TAILQ_FIRST(&list->allio)) != NULL) { TAILQ_REMOVE(&list->allio, tio, links); ctl_free_io(tio->io); @@ -990,10 +980,6 @@ complete: tio->list = list; TAILQ_INSERT_TAIL(&list->allio, tio, links); tio->io = tpcl_alloc_io(); - if (tio->io == NULL) { - list->error = 1; - goto complete; - } ctl_scsi_tur(tio->io, /*tag_type*/ CTL_TAG_SIMPLE, /*control*/ 0); tio->io->io_hdr.retries = 3; tio->lun = sl; @@ -1013,7 +999,6 @@ tpc_process_register_key(struct tpc_list int datalen; if (list->stage == 1) { -complete: while ((tio = TAILQ_FIRST(&list->allio)) != NULL) { TAILQ_REMOVE(&list->allio, tio, links); ctl_free_io(tio->io); @@ -1050,10 +1035,6 @@ complete: tio->list = list; TAILQ_INSERT_TAIL(&list->allio, tio, links); tio->io = tpcl_alloc_io(); - if (tio->io == NULL) { - list->error = 1; - goto complete; - } datalen = sizeof(struct scsi_per_res_out_parms); list->buf = malloc(datalen, M_CTL, M_WAITOK); ctl_scsi_persistent_res_out(tio->io, @@ -1112,7 +1093,6 @@ tpc_process_wut(struct tpc_list *list) uint32_t srcblock, dstblock; if (list->stage > 0) { -complete: /* Cleanup after previous rounds. */ while ((tio = TAILQ_FIRST(&list->allio)) != NULL) { TAILQ_REMOVE(&list->allio, tio, links); @@ -1184,10 +1164,6 @@ complete: tior->list = list; TAILQ_INSERT_TAIL(&list->allio, tior, links); tior->io = tpcl_alloc_io(); - if (tior->io == NULL) { - list->error = 1; - goto complete; - } ctl_scsi_read_write(tior->io, /*data_ptr*/ &list->buf[donebytes], /*data_len*/ roundbytes, @@ -1207,10 +1183,6 @@ complete: tiow->list = list; TAILQ_INSERT_TAIL(&list->allio, tiow, links); tiow->io = tpcl_alloc_io(); - if (tiow->io == NULL) { - list->error = 1; - goto complete; - } ctl_scsi_read_write(tiow->io, /*data_ptr*/ &list->buf[donebytes], /*data_len*/ roundbytes, @@ -1289,10 +1261,6 @@ complete: tiow->list = list; TAILQ_INSERT_TAIL(&list->allio, tiow, links); tiow->io = tpcl_alloc_io(); - if (tiow->io == NULL) { - list->error = 1; - goto complete; - } ctl_scsi_write_same(tiow->io, /*data_ptr*/ list->buf, /*data_len*/ dstblock, Modified: stable/10/sys/cam/ctl/scsi_ctl.c ============================================================================== --- stable/10/sys/cam/ctl/scsi_ctl.c Thu Dec 18 07:38:41 2014 (r275877) +++ stable/10/sys/cam/ctl/scsi_ctl.c Thu Dec 18 08:22:16 2014 (r275878) @@ -72,10 +72,6 @@ __FBSDID("$FreeBSD$"); #include #include -typedef enum { - CTLFE_CCB_DEFAULT = 0x00 -} ctlfe_ccb_types; - struct ctlfe_softc { struct ctl_port port; path_id_t path_id; @@ -189,9 +185,7 @@ SYSCTL_INT(_kern_cam_ctl, OID_AUTO, dma_ &ctlfe_dma_enabled, 0, "DMA enabled"); MALLOC_DEFINE(M_CTLFE, "CAM CTL FE", "CAM CTL FE interface"); -#define ccb_type ppriv_field0 -/* This is only used in the ATIO */ -#define io_ptr ppriv_ptr1 +#define io_ptr ppriv_ptr0 /* This is only used in the CTIO */ #define ccb_atio ppriv_ptr1 @@ -546,6 +540,7 @@ ctlferegister(struct cam_periph *periph, for (i = 0; i < CTLFE_ATIO_PER_LUN; i++) { union ccb *new_ccb; + union ctl_io *new_io; new_ccb = (union ccb *)malloc(sizeof(*new_ccb), M_CTLFE, M_ZERO|M_NOWAIT); @@ -553,6 +548,14 @@ ctlferegister(struct cam_periph *periph, status = CAM_RESRC_UNAVAIL; break; } + new_io = ctl_alloc_io_nowait(bus_softc->port.ctl_pool_ref); + if (new_io == NULL) { + free(new_ccb, M_CTLFE); + status = CAM_RESRC_UNAVAIL; + break; + } + new_ccb->ccb_h.io_ptr = new_io; + xpt_setup_ccb(&new_ccb->ccb_h, periph->path, /*priority*/ 1); new_ccb->ccb_h.func_code = XPT_ACCEPT_TARGET_IO; new_ccb->ccb_h.cbfcnp = ctlfedone; @@ -561,6 +564,7 @@ ctlferegister(struct cam_periph *periph, softc->atios_sent++; status = new_ccb->ccb_h.status; if ((status & CAM_STATUS_MASK) != CAM_REQ_INPROG) { + ctl_free_io(new_io); free(new_ccb, M_CTLFE); break; } @@ -581,6 +585,7 @@ ctlferegister(struct cam_periph *periph, for (i = 0; i < CTLFE_IN_PER_LUN; i++) { union ccb *new_ccb; + union ctl_io *new_io; new_ccb = (union ccb *)malloc(sizeof(*new_ccb), M_CTLFE, M_ZERO|M_NOWAIT); @@ -588,6 +593,13 @@ ctlferegister(struct cam_periph *periph, status = CAM_RESRC_UNAVAIL; break; } + new_io = ctl_alloc_io_nowait(bus_softc->port.ctl_pool_ref); + if (new_io == NULL) { + free(new_ccb, M_CTLFE); + status = CAM_RESRC_UNAVAIL; + break; + } + new_ccb->ccb_h.io_ptr = new_io; xpt_setup_ccb(&new_ccb->ccb_h, periph->path, /*priority*/ 1); new_ccb->ccb_h.func_code = XPT_IMMEDIATE_NOTIFY; @@ -766,8 +778,6 @@ ctlfestart(struct cam_periph *periph, un softc->ccbs_alloced++; - start_ccb->ccb_h.ccb_type = CTLFE_CCB_DEFAULT; - ccb_h = TAILQ_FIRST(&softc->work_queue); if (ccb_h == NULL) { softc->ccbs_freed++; @@ -812,7 +822,6 @@ ctlfestart(struct cam_periph *periph, un xpt_print(periph->path, "%s: aborted " "command 0x%04x discarded\n", __func__, io->scsiio.tag_num); - ctl_free_io(io); /* * For a wildcard attachment, commands can * come in with a specific target/lun. Reset @@ -1038,6 +1047,7 @@ ctlfe_free_ccb(struct cam_periph *periph break; } + ctl_free_io(ccb->ccb_h.io_ptr); free(ccb, M_CTLFE); KASSERT(softc->atios_returned <= softc->atios_sent, ("%s: " @@ -1139,8 +1149,8 @@ ctlfedone(struct cam_periph *periph, uni KASSERT((done_ccb->ccb_h.flags & CAM_UNLOCKED) != 0, ("CCB in ctlfedone() without CAM_UNLOCKED flag")); #ifdef CTLFE_DEBUG - printf("%s: entered, func_code = %#x, type = %#lx\n", __func__, - done_ccb->ccb_h.func_code, done_ccb->ccb_h.ccb_type); + printf("%s: entered, func_code = %#x\n", __func__, + done_ccb->ccb_h.func_code); *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***