From owner-svn-src-stable@FreeBSD.ORG Tue Nov 18 07:16:04 2014 Return-Path: Delivered-To: svn-src-stable@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by hub.freebsd.org (Postfix) with ESMTPS id 445CC505; Tue, 18 Nov 2014 07:16:04 +0000 (UTC) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 305FFAFA; Tue, 18 Nov 2014 07:16:04 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.9/8.14.9) with ESMTP id sAI7G4qJ062019; Tue, 18 Nov 2014 07:16:04 GMT (envelope-from jpaetzel@FreeBSD.org) Received: (from jpaetzel@localhost) by svn.freebsd.org (8.14.9/8.14.9/Submit) id sAI7G4Yv062018; Tue, 18 Nov 2014 07:16:04 GMT (envelope-from jpaetzel@FreeBSD.org) Message-Id: <201411180716.sAI7G4Yv062018@svn.freebsd.org> X-Authentication-Warning: svn.freebsd.org: jpaetzel set sender to jpaetzel@FreeBSD.org using -f From: Josh Paetzel Date: Tue, 18 Nov 2014 07:16:04 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-10@freebsd.org Subject: svn commit: r274646 - stable/10/sys/sys X-SVN-Group: stable-10 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-stable@freebsd.org X-Mailman-Version: 2.1.18-1 Precedence: list List-Id: SVN commit messages for all the -stable branches of the src tree List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 18 Nov 2014 07:16:04 -0000 Author: jpaetzel Date: Tue Nov 18 07:16:03 2014 New Revision: 274646 URL: https://svnweb.freebsd.org/changeset/base/274646 Log: MFC 273866 Plug memory ordering holes in buf_ring_enqueue. For at least some users this patch eliminates the races previously discussed on the mailing list. Submitted by: oleg Reviewed by: kmacy MFC after: 2 weeks Tested by: kmacy,rpaulo Modified: stable/10/sys/sys/buf_ring.h Directory Properties: stable/10/ (props changed) Modified: stable/10/sys/sys/buf_ring.h ============================================================================== --- stable/10/sys/sys/buf_ring.h Tue Nov 18 07:03:10 2014 (r274645) +++ stable/10/sys/sys/buf_ring.h Tue Nov 18 07:16:03 2014 (r274646) @@ -64,8 +64,7 @@ struct buf_ring { static __inline int buf_ring_enqueue(struct buf_ring *br, void *buf) { - uint32_t prod_head, prod_next; - uint32_t cons_tail; + uint32_t prod_head, prod_next, cons_tail; #ifdef DEBUG_BUFRING int i; for (i = br->br_cons_head; i != br->br_prod_head; @@ -77,16 +76,20 @@ buf_ring_enqueue(struct buf_ring *br, vo critical_enter(); do { prod_head = br->br_prod_head; + prod_next = (prod_head + 1) & br->br_prod_mask; cons_tail = br->br_cons_tail; - prod_next = (prod_head + 1) & br->br_prod_mask; - if (prod_next == cons_tail) { - br->br_drops++; - critical_exit(); - return (ENOBUFS); + rmb(); + if (prod_head == br->br_prod_head && + cons_tail == br->br_cons_tail) { + br->br_drops++; + critical_exit(); + return (ENOBUFS); + } + continue; } - } while (!atomic_cmpset_int(&br->br_prod_head, prod_head, prod_next)); + } while (!atomic_cmpset_acq_int(&br->br_prod_head, prod_head, prod_next)); #ifdef DEBUG_BUFRING if (br->br_ring[prod_head] != NULL) panic("dangling value in enqueue"); @@ -94,19 +97,13 @@ buf_ring_enqueue(struct buf_ring *br, vo br->br_ring[prod_head] = buf; /* - * The full memory barrier also avoids that br_prod_tail store - * is reordered before the br_ring[prod_head] is full setup. - */ - mb(); - - /* * If there are other enqueues in progress * that preceeded us, we need to wait for them * to complete */ while (br->br_prod_tail != prod_head) cpu_spinwait(); - br->br_prod_tail = prod_next; + atomic_store_rel_int(&br->br_prod_tail, prod_next); critical_exit(); return (0); } @@ -119,37 +116,23 @@ static __inline void * buf_ring_dequeue_mc(struct buf_ring *br) { uint32_t cons_head, cons_next; - uint32_t prod_tail; void *buf; - int success; critical_enter(); do { cons_head = br->br_cons_head; - prod_tail = br->br_prod_tail; - cons_next = (cons_head + 1) & br->br_cons_mask; - - if (cons_head == prod_tail) { + + if (cons_head == br->br_prod_tail) { critical_exit(); return (NULL); } - - success = atomic_cmpset_int(&br->br_cons_head, cons_head, - cons_next); - } while (success == 0); + } while (!atomic_cmpset_acq_int(&br->br_cons_head, cons_head, cons_next)); buf = br->br_ring[cons_head]; #ifdef DEBUG_BUFRING br->br_ring[cons_head] = NULL; #endif - - /* - * The full memory barrier also avoids that br_ring[cons_read] - * load is reordered after br_cons_tail is set. - */ - mb(); - /* * If there are other dequeues in progress * that preceeded us, we need to wait for them @@ -158,7 +141,7 @@ buf_ring_dequeue_mc(struct buf_ring *br) while (br->br_cons_tail != cons_head) cpu_spinwait(); - br->br_cons_tail = cons_next; + atomic_store_rel_int(&br->br_cons_tail, cons_next); critical_exit(); return (buf);