Date: Wed, 1 Aug 2018 12:49:51 +0000 (UTC) From: Hans Petter Selasky <hselasky@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-10@freebsd.org Subject: svn commit: r337041 - in stable/10/sys: amd64/include i386/include Message-ID: <201808011249.w71CnpWg036475@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: hselasky Date: Wed Aug 1 12:49:51 2018 New Revision: 337041 URL: https://svnweb.freebsd.org/changeset/base/337041 Log: MFC r311169, r312975, r313080, r315718 and r316031: Add support for atomic_(f)cmpset to x86. Modified: stable/10/sys/amd64/include/atomic.h stable/10/sys/i386/include/atomic.h Directory Properties: stable/10/ (props changed) Modified: stable/10/sys/amd64/include/atomic.h ============================================================================== --- stable/10/sys/amd64/include/atomic.h Wed Aug 1 12:18:52 2018 (r337040) +++ stable/10/sys/amd64/include/atomic.h Wed Aug 1 12:49:51 2018 (r337041) @@ -78,8 +78,15 @@ void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); \ void atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v) +int atomic_cmpset_char(volatile u_char *dst, u_char expect, u_char src); +int atomic_cmpset_short(volatile u_short *dst, u_short expect, u_short src); int atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src); int atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src); +int atomic_fcmpset_char(volatile u_char *dst, u_char *expect, u_char src); +int atomic_fcmpset_short(volatile u_short *dst, u_short *expect, + u_short src); +int atomic_fcmpset_int(volatile u_int *dst, u_int *expect, u_int src); +int atomic_fcmpset_long(volatile u_long *dst, u_long *expect, u_long src); u_int atomic_fetchadd_int(volatile u_int *p, u_int v); u_long atomic_fetchadd_long(volatile u_long *p, u_long v); int atomic_testandset_int(volatile u_int *p, u_int v); @@ -130,49 +137,62 @@ atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_## struct __hack /* - * Atomic compare and set, used by the mutex functions + * Atomic compare and set, used by the mutex functions. * - * if (*dst == expect) *dst = src (all 32 bit words) + * cmpset: + * if (*dst == expect) + * *dst = src * - * Returns 0 on failure, non-zero on success + * fcmpset: + * if (*dst == *expect) + * *dst = src + * else + * *expect = *dst + * + * Returns 0 on failure, non-zero on success. */ - -static __inline int -atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src) -{ - u_char res; - - __asm __volatile( - " " MPLOCKED " " - " cmpxchgl %3,%1 ; " - " sete %0 ; " - "# atomic_cmpset_int" - : "=q" (res), /* 0 */ - "+m" (*dst), /* 1 */ - "+a" (expect) /* 2 */ - : "r" (src) /* 3 */ - : "memory", "cc"); - return (res); +#define ATOMIC_CMPSET(TYPE) \ +static __inline int \ +atomic_cmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE expect, u_##TYPE src) \ +{ \ + u_char res; \ + \ + __asm __volatile( \ + " " MPLOCKED " " \ + " cmpxchg %3,%1 ; " \ + " sete %0 ; " \ + "# atomic_cmpset_" #TYPE " " \ + : "=q" (res), /* 0 */ \ + "+m" (*dst), /* 1 */ \ + "+a" (expect) /* 2 */ \ + : "r" (src) /* 3 */ \ + : "memory", "cc"); \ + return (res); \ +} \ + \ +static __inline int \ +atomic_fcmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE *expect, u_##TYPE src) \ +{ \ + u_char res; \ + \ + __asm __volatile( \ + " " MPLOCKED " " \ + " cmpxchg %3,%1 ; " \ + " sete %0 ; " \ + "# atomic_fcmpset_" #TYPE " " \ + : "=q" (res), /* 0 */ \ + "+m" (*dst), /* 1 */ \ + "+a" (*expect) /* 2 */ \ + : "r" (src) /* 3 */ \ + : "memory", "cc"); \ + return (res); \ } -static __inline int -atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src) -{ - u_char res; +ATOMIC_CMPSET(char); +ATOMIC_CMPSET(short); +ATOMIC_CMPSET(int); +ATOMIC_CMPSET(long); - __asm __volatile( - " " MPLOCKED " " - " cmpxchgq %3,%1 ; " - " sete %0 ; " - "# atomic_cmpset_long" - : "=q" (res), /* 0 */ - "+m" (*dst), /* 1 */ - "+a" (expect) /* 2 */ - : "r" (src) /* 3 */ - : "memory", "cc"); - return (res); -} - /* * Atomically add the value of v to the integer pointed to by p and return * the previous value of *p. @@ -409,6 +429,10 @@ u_long atomic_swap_long(volatile u_long *p, u_long v); #define atomic_add_rel_char atomic_add_barr_char #define atomic_subtract_acq_char atomic_subtract_barr_char #define atomic_subtract_rel_char atomic_subtract_barr_char +#define atomic_cmpset_acq_char atomic_cmpset_char +#define atomic_cmpset_rel_char atomic_cmpset_char +#define atomic_fcmpset_acq_char atomic_fcmpset_char +#define atomic_fcmpset_rel_char atomic_fcmpset_char #define atomic_set_acq_short atomic_set_barr_short #define atomic_set_rel_short atomic_set_barr_short @@ -418,6 +442,10 @@ u_long atomic_swap_long(volatile u_long *p, u_long v); #define atomic_add_rel_short atomic_add_barr_short #define atomic_subtract_acq_short atomic_subtract_barr_short #define atomic_subtract_rel_short atomic_subtract_barr_short +#define atomic_cmpset_acq_short atomic_cmpset_short +#define atomic_cmpset_rel_short atomic_cmpset_short +#define atomic_fcmpset_acq_short atomic_fcmpset_short +#define atomic_fcmpset_rel_short atomic_fcmpset_short #define atomic_set_acq_int atomic_set_barr_int #define atomic_set_rel_int atomic_set_barr_int @@ -429,6 +457,8 @@ u_long atomic_swap_long(volatile u_long *p, u_long v); #define atomic_subtract_rel_int atomic_subtract_barr_int #define atomic_cmpset_acq_int atomic_cmpset_int #define atomic_cmpset_rel_int atomic_cmpset_int +#define atomic_fcmpset_acq_int atomic_fcmpset_int +#define atomic_fcmpset_rel_int atomic_fcmpset_int #define atomic_set_acq_long atomic_set_barr_long #define atomic_set_rel_long atomic_set_barr_long @@ -440,6 +470,8 @@ u_long atomic_swap_long(volatile u_long *p, u_long v); #define atomic_subtract_rel_long atomic_subtract_barr_long #define atomic_cmpset_acq_long atomic_cmpset_long #define atomic_cmpset_rel_long atomic_cmpset_long +#define atomic_fcmpset_acq_long atomic_fcmpset_long +#define atomic_fcmpset_rel_long atomic_fcmpset_long #define atomic_readandclear_int(p) atomic_swap_int(p, 0) #define atomic_readandclear_long(p) atomic_swap_long(p, 0) @@ -459,6 +491,12 @@ u_long atomic_swap_long(volatile u_long *p, u_long v); #define atomic_subtract_rel_8 atomic_subtract_rel_char #define atomic_load_acq_8 atomic_load_acq_char #define atomic_store_rel_8 atomic_store_rel_char +#define atomic_cmpset_8 atomic_cmpset_char +#define atomic_cmpset_acq_8 atomic_cmpset_acq_char +#define atomic_cmpset_rel_8 atomic_cmpset_rel_char +#define atomic_fcmpset_8 atomic_fcmpset_char +#define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char +#define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char /* Operations on 16-bit words. */ #define atomic_set_16 atomic_set_short @@ -475,6 +513,12 @@ u_long atomic_swap_long(volatile u_long *p, u_long v); #define atomic_subtract_rel_16 atomic_subtract_rel_short #define atomic_load_acq_16 atomic_load_acq_short #define atomic_store_rel_16 atomic_store_rel_short +#define atomic_cmpset_16 atomic_cmpset_short +#define atomic_cmpset_acq_16 atomic_cmpset_acq_short +#define atomic_cmpset_rel_16 atomic_cmpset_rel_short +#define atomic_fcmpset_16 atomic_fcmpset_short +#define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short +#define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short /* Operations on 32-bit double words. */ #define atomic_set_32 atomic_set_int @@ -494,6 +538,9 @@ u_long atomic_swap_long(volatile u_long *p, u_long v); #define atomic_cmpset_32 atomic_cmpset_int #define atomic_cmpset_acq_32 atomic_cmpset_acq_int #define atomic_cmpset_rel_32 atomic_cmpset_rel_int +#define atomic_fcmpset_32 atomic_fcmpset_int +#define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int +#define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int #define atomic_swap_32 atomic_swap_int #define atomic_readandclear_32 atomic_readandclear_int #define atomic_fetchadd_32 atomic_fetchadd_int @@ -518,6 +565,9 @@ u_long atomic_swap_long(volatile u_long *p, u_long v); #define atomic_cmpset_64 atomic_cmpset_long #define atomic_cmpset_acq_64 atomic_cmpset_acq_long #define atomic_cmpset_rel_64 atomic_cmpset_rel_long +#define atomic_fcmpset_64 atomic_fcmpset_long +#define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long +#define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long #define atomic_swap_64 atomic_swap_long #define atomic_readandclear_64 atomic_readandclear_long #define atomic_fetchadd_64 atomic_fetchadd_long @@ -542,6 +592,9 @@ u_long atomic_swap_long(volatile u_long *p, u_long v); #define atomic_cmpset_ptr atomic_cmpset_long #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long +#define atomic_fcmpset_ptr atomic_fcmpset_long +#define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long +#define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long #define atomic_swap_ptr atomic_swap_long #define atomic_readandclear_ptr atomic_readandclear_long Modified: stable/10/sys/i386/include/atomic.h ============================================================================== --- stable/10/sys/i386/include/atomic.h Wed Aug 1 12:18:52 2018 (r337040) +++ stable/10/sys/i386/include/atomic.h Wed Aug 1 12:49:51 2018 (r337041) @@ -83,7 +83,13 @@ void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); \ void atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v) +int atomic_cmpset_char(volatile u_char *dst, u_char expect, u_char src); +int atomic_cmpset_short(volatile u_short *dst, u_short expect, u_short src); int atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src); +int atomic_fcmpset_char(volatile u_char *dst, u_char *expect, u_char src); +int atomic_fcmpset_short(volatile u_short *dst, u_short *expect, + u_short src); +int atomic_fcmpset_int(volatile u_int *dst, u_int *expect, u_int src); u_int atomic_fetchadd_int(volatile u_int *p, u_int v); int atomic_testandset_int(volatile u_int *p, u_int v); int atomic_testandclear_int(volatile u_int *p, u_int v); @@ -137,60 +143,61 @@ atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_## struct __hack /* - * Atomic compare and set, used by the mutex functions + * Atomic compare and set, used by the mutex functions. * - * if (*dst == expect) *dst = src (all 32 bit words) + * cmpset: + * if (*dst == expect) + * *dst = src * - * Returns 0 on failure, non-zero on success + * fcmpset: + * if (*dst == *expect) + * *dst = src + * else + * *expect = *dst + * + * Returns 0 on failure, non-zero on success. */ - -#ifdef CPU_DISABLE_CMPXCHG - -static __inline int -atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src) -{ - u_char res; - - __asm __volatile( - " pushfl ; " - " cli ; " - " cmpl %3,%1 ; " - " jne 1f ; " - " movl %2,%1 ; " - "1: " - " sete %0 ; " - " popfl ; " - "# atomic_cmpset_int" - : "=q" (res), /* 0 */ - "+m" (*dst) /* 1 */ - : "r" (src), /* 2 */ - "r" (expect) /* 3 */ - : "memory"); - return (res); +#define ATOMIC_CMPSET(TYPE, CONS) \ +static __inline int \ +atomic_cmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE expect, u_##TYPE src) \ +{ \ + u_char res; \ + \ + __asm __volatile( \ + " " MPLOCKED " " \ + " cmpxchg %3,%1 ; " \ + " sete %0 ; " \ + "# atomic_cmpset_" #TYPE " " \ + : "=q" (res), /* 0 */ \ + "+m" (*dst), /* 1 */ \ + "+a" (expect) /* 2 */ \ + : CONS (src) /* 3 */ \ + : "memory", "cc"); \ + return (res); \ +} \ + \ +static __inline int \ +atomic_fcmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE *expect, u_##TYPE src) \ +{ \ + u_char res; \ + \ + __asm __volatile( \ + " " MPLOCKED " " \ + " cmpxchg %3,%1 ; " \ + " sete %0 ; " \ + "# atomic_fcmpset_" #TYPE " " \ + : "=q" (res), /* 0 */ \ + "+m" (*dst), /* 1 */ \ + "+a" (*expect) /* 2 */ \ + : CONS (src) /* 3 */ \ + : "memory", "cc"); \ + return (res); \ } -#else /* !CPU_DISABLE_CMPXCHG */ +ATOMIC_CMPSET(char, "q"); +ATOMIC_CMPSET(short, "r"); +ATOMIC_CMPSET(int, "r"); -static __inline int -atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src) -{ - u_char res; - - __asm __volatile( - " " MPLOCKED " " - " cmpxchgl %3,%1 ; " - " sete %0 ; " - "# atomic_cmpset_int" - : "=q" (res), /* 0 */ - "+m" (*dst), /* 1 */ - "+a" (expect) /* 2 */ - : "r" (src) /* 3 */ - : "memory", "cc"); - return (res); -} - -#endif /* CPU_DISABLE_CMPXCHG */ - /* * Atomically add the value of v to the integer pointed to by p and return * the previous value of *p. @@ -623,6 +630,10 @@ u_long atomic_swap_long(volatile u_long *p, u_long v); #define atomic_add_rel_char atomic_add_barr_char #define atomic_subtract_acq_char atomic_subtract_barr_char #define atomic_subtract_rel_char atomic_subtract_barr_char +#define atomic_cmpset_acq_char atomic_cmpset_char +#define atomic_cmpset_rel_char atomic_cmpset_char +#define atomic_fcmpset_acq_char atomic_fcmpset_char +#define atomic_fcmpset_rel_char atomic_fcmpset_char #define atomic_set_acq_short atomic_set_barr_short #define atomic_set_rel_short atomic_set_barr_short @@ -632,6 +643,10 @@ u_long atomic_swap_long(volatile u_long *p, u_long v); #define atomic_add_rel_short atomic_add_barr_short #define atomic_subtract_acq_short atomic_subtract_barr_short #define atomic_subtract_rel_short atomic_subtract_barr_short +#define atomic_cmpset_acq_short atomic_cmpset_short +#define atomic_cmpset_rel_short atomic_cmpset_short +#define atomic_fcmpset_acq_short atomic_fcmpset_short +#define atomic_fcmpset_rel_short atomic_fcmpset_short #define atomic_set_acq_int atomic_set_barr_int #define atomic_set_rel_int atomic_set_barr_int @@ -643,6 +658,8 @@ u_long atomic_swap_long(volatile u_long *p, u_long v); #define atomic_subtract_rel_int atomic_subtract_barr_int #define atomic_cmpset_acq_int atomic_cmpset_int #define atomic_cmpset_rel_int atomic_cmpset_int +#define atomic_fcmpset_acq_int atomic_fcmpset_int +#define atomic_fcmpset_rel_int atomic_fcmpset_int #define atomic_set_acq_long atomic_set_barr_long #define atomic_set_rel_long atomic_set_barr_long @@ -654,6 +671,8 @@ u_long atomic_swap_long(volatile u_long *p, u_long v); #define atomic_subtract_rel_long atomic_subtract_barr_long #define atomic_cmpset_acq_long atomic_cmpset_long #define atomic_cmpset_rel_long atomic_cmpset_long +#define atomic_fcmpset_acq_long atomic_fcmpset_long +#define atomic_fcmpset_rel_long atomic_fcmpset_long #define atomic_readandclear_int(p) atomic_swap_int(p, 0) #define atomic_readandclear_long(p) atomic_swap_long(p, 0) @@ -673,6 +692,12 @@ u_long atomic_swap_long(volatile u_long *p, u_long v); #define atomic_subtract_rel_8 atomic_subtract_rel_char #define atomic_load_acq_8 atomic_load_acq_char #define atomic_store_rel_8 atomic_store_rel_char +#define atomic_cmpset_8 atomic_cmpset_char +#define atomic_cmpset_acq_8 atomic_cmpset_acq_char +#define atomic_cmpset_rel_8 atomic_cmpset_rel_char +#define atomic_fcmpset_8 atomic_fcmpset_char +#define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char +#define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char /* Operations on 16-bit words. */ #define atomic_set_16 atomic_set_short @@ -689,6 +714,12 @@ u_long atomic_swap_long(volatile u_long *p, u_long v); #define atomic_subtract_rel_16 atomic_subtract_rel_short #define atomic_load_acq_16 atomic_load_acq_short #define atomic_store_rel_16 atomic_store_rel_short +#define atomic_cmpset_16 atomic_cmpset_short +#define atomic_cmpset_acq_16 atomic_cmpset_acq_short +#define atomic_cmpset_rel_16 atomic_cmpset_rel_short +#define atomic_fcmpset_16 atomic_fcmpset_short +#define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short +#define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short /* Operations on 32-bit double words. */ #define atomic_set_32 atomic_set_int @@ -708,6 +739,9 @@ u_long atomic_swap_long(volatile u_long *p, u_long v); #define atomic_cmpset_32 atomic_cmpset_int #define atomic_cmpset_acq_32 atomic_cmpset_acq_int #define atomic_cmpset_rel_32 atomic_cmpset_rel_int +#define atomic_fcmpset_32 atomic_fcmpset_int +#define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int +#define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int #define atomic_swap_32 atomic_swap_int #define atomic_readandclear_32 atomic_readandclear_int #define atomic_fetchadd_32 atomic_fetchadd_int @@ -750,6 +784,14 @@ u_long atomic_swap_long(volatile u_long *p, u_long v); (u_int)(new)) #define atomic_cmpset_rel_ptr(dst, old, new) \ atomic_cmpset_rel_int((volatile u_int *)(dst), (u_int)(old), \ + (u_int)(new)) +#define atomic_fcmpset_ptr(dst, old, new) \ + atomic_fcmpset_int((volatile u_int *)(dst), (u_int *)(old), (u_int)(new)) +#define atomic_fcmpset_acq_ptr(dst, old, new) \ + atomic_fcmpset_acq_int((volatile u_int *)(dst), (u_int *)(old), \ + (u_int)(new)) +#define atomic_fcmpset_rel_ptr(dst, old, new) \ + atomic_fcmpset_rel_int((volatile u_int *)(dst), (u_int *)(old), \ (u_int)(new)) #define atomic_swap_ptr(p, v) \ atomic_swap_int((volatile u_int *)(p), (u_int)(v))
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201808011249.w71CnpWg036475>