Date: Fri, 17 Jun 2005 19:55:47 GMT From: John Baldwin <jhb@FreeBSD.org> To: Perforce Change Reviews <perforce@freebsd.org> Subject: PERFORCE change 78683 for review Message-ID: <200506171955.j5HJtlST050915@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=78683 Change 78683 by jhb@jhb_slimer on 2005/06/17 19:55:36 Backout the atomic patches for now so I can break it up into more palatable chunks. Affected files ... .. //depot/projects/smpng/sys/alpha/include/atomic.h#21 edit .. //depot/projects/smpng/sys/amd64/include/atomic.h#15 edit .. //depot/projects/smpng/sys/amd64/include/bus.h#7 edit .. //depot/projects/smpng/sys/arm/include/atomic.h#11 edit .. //depot/projects/smpng/sys/arm/include/endian.h#11 edit .. //depot/projects/smpng/sys/dev/hatm/if_hatm_intr.c#12 edit .. //depot/projects/smpng/sys/i386/include/atomic.h#32 edit .. //depot/projects/smpng/sys/i386/include/bus.h#8 edit .. //depot/projects/smpng/sys/ia64/include/atomic.h#9 edit .. //depot/projects/smpng/sys/kern/kern_mutex.c#99 edit .. //depot/projects/smpng/sys/pc98/include/bus.h#4 edit .. //depot/projects/smpng/sys/powerpc/include/atomic.h#12 edit .. //depot/projects/smpng/sys/sparc64/include/atomic.h#12 edit .. //depot/projects/smpng/sys/sparc64/include/cpufunc.h#24 edit .. //depot/projects/smpng/sys/sys/mutex.h#56 edit Differences ... ==== //depot/projects/smpng/sys/alpha/include/atomic.h#21 (text+ko) ==== @@ -56,12 +56,12 @@ #ifdef __GNUCLIKE_ASM __asm __volatile ( - "1:\tldl_l %0, %1\n\t" /* load old value */ - "bis %0, %2, %0\n\t" /* calculate new value */ + "1:\tldl_l %0, %2\n\t" /* load old value */ + "bis %0, %3, %0\n\t" /* calculate new value */ "stl_c %0, %1\n\t" /* attempt to store */ "beq %0, 1b\n" /* spin if failed */ : "=&r" (temp), "=m" (*p) - : "r" (v), "m" (*p) + : "m" (*p), "r" (v) : "memory"); #endif } @@ -76,8 +76,8 @@ "bic %0, %2, %0\n\t" /* calculate new value */ "stl_c %0, %1\n\t" /* attempt to store */ "beq %0, 1b\n" /* spin if failed */ - : "=&r" (temp), "=m" (*p) - : "r" (v), "m" (*p) + : "=&r" (temp), "+m" (*p) + : "r" (v) : "memory"); #endif } @@ -92,8 +92,8 @@ "addl %0, %2, %0\n\t" /* calculate new value */ "stl_c %0, %1\n\t" /* attempt to store */ "beq %0, 1b\n" /* spin if failed */ - : "=&r" (temp), "=m" (*p) - : "r" (v), "m" (*p) + : "=&r" (temp), "+m" (*p) + : "r" (v) : "memory"); #endif } @@ -108,8 +108,8 @@ "subl %0, %2, %0\n\t" /* calculate new value */ "stl_c %0, %1\n\t" /* attempt to store */ "beq %0, 1b\n" /* spin if failed */ - : "=&r" (temp), "=m" (*p) - : "r" (v), "m" (*p) + : "=&r" (temp), "+m" (*p) + : "r" (v) : "memory"); #endif } @@ -125,8 +125,8 @@ "ldiq %1,0\n\t" /* value to store */ "stl_c %1,%2\n\t" /* attempt to store */ "beq %1,1b\n" /* if the store failed, spin */ - : "=&r"(result), "=&r"(temp), "=m" (*addr) - : "m" (*addr) + : "=&r"(result), "=&r"(temp), "+m" (*addr) + : : "memory"); #endif @@ -143,8 +143,8 @@ "bis %0, %2, %0\n\t" /* calculate new value */ "stq_c %0, %1\n\t" /* attempt to store */ "beq %0, 1b\n" /* spin if failed */ - : "=&r" (temp), "=m" (*p) - : "r" (v), "m" (*p) + : "=&r" (temp), "+m" (*p) + : "r" (v) : "memory"); #endif } @@ -159,8 +159,8 @@ "bic %0, %2, %0\n\t" /* calculate new value */ "stq_c %0, %1\n\t" /* attempt to store */ "beq %0, 1b\n" /* spin if failed */ - : "=&r" (temp), "=m" (*p) - : "r" (v), "m" (*p) + : "=&r" (temp), "+m" (*p) + : "r" (v) : "memory"); #endif } @@ -175,8 +175,8 @@ "addq %0, %2, %0\n\t" /* calculate new value */ "stq_c %0, %1\n\t" /* attempt to store */ "beq %0, 1b\n" /* spin if failed */ - : "=&r" (temp), "=m" (*p) - : "r" (v), "m" (*p) + : "=&r" (temp), "+m" (*p) + : "r" (v) : "memory"); #endif } @@ -191,8 +191,8 @@ "subq %0, %2, %0\n\t" /* calculate new value */ "stq_c %0, %1\n\t" /* attempt to store */ "beq %0, 1b\n" /* spin if failed */ - : "=&r" (temp), "=m" (*p) - : "r" (v), "m" (*p) + : "=&r" (temp), "+m" (*p) + : "r" (v) : "memory"); #endif } @@ -208,14 +208,36 @@ "ldiq %1,0\n\t" /* value to store */ "stq_c %1,%2\n\t" /* attempt to store */ "beq %1,1b\n" /* if the store failed, spin */ - : "=&r"(result), "=&r"(temp), "=m" (*addr) - : "m" (*addr) + : "=&r"(result), "=&r"(temp), "+m" (*addr) + : : "memory"); #endif return result; } +#define atomic_set_char atomic_set_8 +#define atomic_clear_char atomic_clear_8 +#define atomic_add_char atomic_add_8 +#define atomic_subtract_char atomic_subtract_8 + +#define atomic_set_short atomic_set_16 +#define atomic_clear_short atomic_clear_16 +#define atomic_add_short atomic_add_16 +#define atomic_subtract_short atomic_subtract_16 + +#define atomic_set_int atomic_set_32 +#define atomic_clear_int atomic_clear_32 +#define atomic_add_int atomic_add_32 +#define atomic_subtract_int atomic_subtract_32 +#define atomic_readandclear_int atomic_readandclear_32 + +#define atomic_set_long atomic_set_64 +#define atomic_clear_long atomic_clear_64 +#define atomic_add_long atomic_add_64 +#define atomic_subtract_long atomic_subtract_64 +#define atomic_readandclear_long atomic_readandclear_64 + #define ATOMIC_ACQ_REL(NAME, WIDTH, TYPE) \ static __inline void \ atomic_##NAME##_acq_##WIDTH(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v)\ @@ -227,11 +249,24 @@ static __inline void \ atomic_##NAME##_rel_##WIDTH(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v)\ { \ - alpha_wmb(); \ + alpha_mb(); \ + atomic_##NAME##_##WIDTH(p, v); \ +} \ + \ +static __inline void \ +atomic_##NAME##_acq_##TYPE(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v)\ +{ \ + atomic_##NAME##_##WIDTH(p, v); \ + alpha_mb(); \ +} \ + \ +static __inline void \ +atomic_##NAME##_rel_##TYPE(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v)\ +{ \ + alpha_mb(); \ atomic_##NAME##_##WIDTH(p, v); \ } -/* Variants of simple arithmetic with memory barriers. */ ATOMIC_ACQ_REL(set, 8, char) ATOMIC_ACQ_REL(clear, 8, char) ATOMIC_ACQ_REL(add, 8, char) @@ -254,11 +289,27 @@ /* * We assume that a = b will do atomic loads and stores. */ -#define ATOMIC_STORE_LOAD(WIDTH) \ -static __inline u_int##WIDTH##_t \ -atomic_load_acq_##WIDTH(volatile u_int##WIDTH##_t *p) \ +#define ATOMIC_STORE_LOAD(TYPE, WIDTH) \ +static __inline u_##TYPE \ +atomic_load_acq_##WIDTH(volatile u_##TYPE *p) \ +{ \ + u_##TYPE v; \ + \ + v = *p; \ + alpha_mb(); \ + return (v); \ +} \ + \ +static __inline void \ +atomic_store_rel_##WIDTH(volatile u_##TYPE *p, u_##TYPE v)\ +{ \ + alpha_mb(); \ + *p = v; \ +} \ +static __inline u_##TYPE \ +atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ { \ - u_int##WIDTH##_t v; \ + u_##TYPE v; \ \ v = *p; \ alpha_mb(); \ @@ -266,14 +317,16 @@ } \ \ static __inline void \ -atomic_store_rel_##WIDTH(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v)\ +atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ { \ - alpha_wmb(); \ + alpha_mb(); \ *p = v; \ } -ATOMIC_STORE_LOAD(32) -ATOMIC_STORE_LOAD(64) +ATOMIC_STORE_LOAD(char, 8) +ATOMIC_STORE_LOAD(short, 16) +ATOMIC_STORE_LOAD(int, 32) +ATOMIC_STORE_LOAD(long, 64) #undef ATOMIC_STORE_LOAD @@ -296,8 +349,8 @@ "stl_c %0, %1\n\t" /* attempt to store */ "beq %0, 1b\n\t" /* if it failed, spin */ "2:\n" - : "=&r" (ret), "=m" (*p) - : "r" ((long)(int)cmpval), "r" (newval), "m" (*p) + : "=&r" (ret), "+m" (*p) + : "r" ((long)(int)cmpval), "r" (newval) : "memory"); #endif @@ -323,14 +376,25 @@ "stq_c %0, %1\n\t" /* attempt to store */ "beq %0, 1b\n\t" /* if it failed, spin */ "2:\n" - : "=&r" (ret), "=m" (*p) - : "r" (cmpval), "r" (newval), "m" (*p) + : "=&r" (ret), "+m" (*p) + : "r" (cmpval), "r" (newval) : "memory"); #endif return ret; } +#define atomic_cmpset_int atomic_cmpset_32 +#define atomic_cmpset_long atomic_cmpset_64 + +static __inline int +atomic_cmpset_ptr(volatile void *dst, void *exp, void *src) +{ + + return (atomic_cmpset_long((volatile u_long *)dst, (u_long)exp, + (u_long)src)); +} + static __inline u_int32_t atomic_cmpset_acq_32(volatile u_int32_t *p, u_int32_t cmpval, u_int32_t newval) { @@ -344,7 +408,7 @@ static __inline u_int32_t atomic_cmpset_rel_32(volatile u_int32_t *p, u_int32_t cmpval, u_int32_t newval) { - alpha_wmb(); + alpha_mb(); return (atomic_cmpset_32(p, cmpval, newval)); } @@ -361,96 +425,67 @@ static __inline u_int64_t atomic_cmpset_rel_64(volatile u_int64_t *p, u_int64_t cmpval, u_int64_t newval) { - alpha_wmb(); + alpha_mb(); return (atomic_cmpset_64(p, cmpval, newval)); } -/* Operations on chars. */ -#define atomic_set_char atomic_set_8 -#define atomic_set_acq_char atomic_set_acq_8 -#define atomic_set_rel_char atomic_set_rel_8 -#define atomic_clear_char atomic_clear_8 -#define atomic_clear_acq_char atomic_clear_acq_8 -#define atomic_clear_rel_char atomic_clear_rel_8 -#define atomic_add_char atomic_add_8 -#define atomic_add_acq_char atomic_add_acq_8 -#define atomic_add_rel_char atomic_add_rel_8 -#define atomic_subtract_char atomic_subtract_8 -#define atomic_subtract_acq_char atomic_subtract_acq_8 -#define atomic_subtract_rel_char atomic_subtract_rel_8 - -/* Operations on shorts. */ -#define atomic_set_short atomic_set_16 -#define atomic_set_acq_short atomic_set_acq_16 -#define atomic_set_rel_short atomic_set_rel_16 -#define atomic_clear_short atomic_clear_16 -#define atomic_clear_acq_short atomic_clear_acq_16 -#define atomic_clear_rel_short atomic_clear_rel_16 -#define atomic_add_short atomic_add_16 -#define atomic_add_acq_short atomic_add_acq_16 -#define atomic_add_rel_short atomic_add_rel_16 -#define atomic_subtract_short atomic_subtract_16 -#define atomic_subtract_acq_short atomic_subtract_acq_16 -#define atomic_subtract_rel_short atomic_subtract_rel_16 - -/* Operations on ints. */ -#define atomic_set_int atomic_set_32 -#define atomic_set_acq_int atomic_set_acq_32 -#define atomic_set_rel_int atomic_set_rel_32 -#define atomic_clear_int atomic_clear_32 -#define atomic_clear_acq_int atomic_clear_acq_32 -#define atomic_clear_rel_int atomic_clear_rel_32 -#define atomic_add_int atomic_add_32 -#define atomic_add_acq_int atomic_add_acq_32 -#define atomic_add_rel_int atomic_add_rel_32 -#define atomic_subtract_int atomic_subtract_32 -#define atomic_subtract_acq_int atomic_subtract_acq_32 -#define atomic_subtract_rel_int atomic_subtract_rel_32 -#define atomic_cmpset_int atomic_cmpset_32 #define atomic_cmpset_acq_int atomic_cmpset_acq_32 #define atomic_cmpset_rel_int atomic_cmpset_rel_32 -#define atomic_load_acq_int atomic_load_acq_32 -#define atomic_store_rel_int atomic_store_rel_32 -#define atomic_readandclear_int atomic_readandclear_32 - -/* Operations on longs. */ -#define atomic_set_long atomic_set_64 -#define atomic_set_acq_long atomic_set_acq_64 -#define atomic_set_rel_long atomic_set_rel_64 -#define atomic_clear_long atomic_clear_64 -#define atomic_clear_acq_long atomic_clear_acq_64 -#define atomic_clear_rel_long atomic_clear_rel_64 -#define atomic_add_long atomic_add_64 -#define atomic_add_acq_long atomic_add_acq_64 -#define atomic_add_rel_long atomic_add_rel_64 -#define atomic_subtract_long atomic_subtract_64 -#define atomic_subtract_acq_long atomic_subtract_acq_64 -#define atomic_subtract_rel_long atomic_subtract_rel_64 -#define atomic_cmpset_long atomic_cmpset_64 #define atomic_cmpset_acq_long atomic_cmpset_acq_64 #define atomic_cmpset_rel_long atomic_cmpset_rel_64 -#define atomic_load_acq_long atomic_load_acq_64 -#define atomic_store_rel_long atomic_store_rel_64 -#define atomic_readandclear_long atomic_readandclear_64 + +static __inline int +atomic_cmpset_acq_ptr(volatile void *dst, void *exp, void *src) +{ + + return (atomic_cmpset_acq_long((volatile u_long *)dst, (u_long)exp, + (u_long)src)); +} + +static __inline int +atomic_cmpset_rel_ptr(volatile void *dst, void *exp, void *src) +{ + + return (atomic_cmpset_rel_long((volatile u_long *)dst, (u_long)exp, + (u_long)src)); +} + +static __inline void * +atomic_load_acq_ptr(volatile void *p) +{ + return (void *)atomic_load_acq_long((volatile u_long *)p); +} + +static __inline void +atomic_store_rel_ptr(volatile void *p, void *v) +{ + atomic_store_rel_long((volatile u_long *)p, (u_long)v); +} + +#define ATOMIC_PTR(NAME) \ +static __inline void \ +atomic_##NAME##_ptr(volatile void *p, uintptr_t v) \ +{ \ + atomic_##NAME##_long((volatile u_long *)p, v); \ +} \ + \ +static __inline void \ +atomic_##NAME##_acq_ptr(volatile void *p, uintptr_t v) \ +{ \ + atomic_##NAME##_acq_long((volatile u_long *)p, v);\ +} \ + \ +static __inline void \ +atomic_##NAME##_rel_ptr(volatile void *p, uintptr_t v) \ +{ \ + atomic_##NAME##_rel_long((volatile u_long *)p, v);\ +} + +ATOMIC_PTR(set) +ATOMIC_PTR(clear) +ATOMIC_PTR(add) +ATOMIC_PTR(subtract) -/* Operations on pointers. */ -#define atomic_set_ptr atomic_set_64 -#define atomic_set_acq_ptr atomic_set_acq_64 -#define atomic_set_rel_ptr atomic_set_rel_64 -#define atomic_clear_ptr atomic_clear_64 -#define atomic_clear_acq_ptr atomic_clear_acq_64 -#define atomic_clear_rel_ptr atomic_clear_rel_64 -#define atomic_add_ptr atomic_add_64 -#define atomic_add_acq_ptr atomic_add_acq_64 -#define atomic_add_rel_ptr atomic_add_rel_64 -#define atomic_subtract_ptr atomic_subtract_64 -#define atomic_subtract_acq_ptr atomic_subtract_acq_64 -#define atomic_subtract_rel_ptr atomic_subtract_rel_64 -#define atomic_cmpset_ptr atomic_cmpset_64 -#define atomic_cmpset_acq_ptr atomic_cmpset_acq_64 -#define atomic_cmpset_rel_ptr atomic_cmpset_rel_64 -#define atomic_load_acq_ptr atomic_load_acq_64 -#define atomic_store_rel_ptr atomic_store_rel_64 -#define atomic_readandclear_ptr atomic_readandclear_64 +#undef ATOMIC_PTR #endif /* ! _MACHINE_ATOMIC_H_ */ ==== //depot/projects/smpng/sys/amd64/include/atomic.h#15 (text+ko) ==== @@ -68,15 +68,13 @@ * This allows kernel modules to be portable between UP and SMP systems. */ #if defined(KLD_MODULE) -#define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ -void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); \ -void atomic_##NAME##_##TYPE_acq(volatile u_##TYPE *p, u_##TYPE v); \ -void atomic_##NAME##_##TYPE_rel(volatile u_##TYPE *p, u_##TYPE v) +#define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ +void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v) int atomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src); int atomic_cmpset_long(volatile u_long *dst, u_long exp, u_long src); -#define ATOMIC_STORE_LOAD(TYPE) \ +#define ATOMIC_STORE_LOAD(TYPE, LOP, SOP) \ u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p); \ void atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) @@ -89,13 +87,9 @@ * the binaries will run on both types of systems. */ #if defined(SMP) || !defined(_KERNEL) -#define MPLOCKED "lock ; " -#define MPLFENCE "lfence ; " -#define MPSFENCE "sfence ; " +#define MPLOCKED lock ; #else -#define MPLOCKED "" -#define MPLFENCE "" -#define MPSFENCE "" +#define MPLOCKED #endif /* @@ -106,32 +100,16 @@ static __inline void \ atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ { \ - __asm __volatile(MPLOCKED OP \ - : "=m" (*p) \ - : CONS (V), "m" (*p)); \ -} \ - \ -static __inline void \ -atomic_##NAME##_##TYPE##_acq(volatile u_##TYPE *p, u_##TYPE v)\ -{ \ - atomic_##NAME_##TYPE(p, v); \ - __asm __volatile(MPLFENCE ::: "memory"); \ + __asm __volatile(__XSTRING(MPLOCKED) OP \ + : "+m" (*p) \ + : CONS (V)); \ } \ - \ -static __inline void \ -atomic_##NAME##_##TYPE##_rel(volatile u_##TYPE *p, u_##TYPE v)\ -{ \ - __asm __volatile(MPSFENCE); \ - atomic_##NAME_##TYPE(p, v); \ -} \ struct __hack #else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */ #define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ -extern void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); \ -extern void atomic_##NAME##_##TYPE##_acq(volatile u_##TYPE *p, u_##TYPE v); \ -extern void atomic_##NAME##_##TYPE##_rel(volatile u_##TYPE *p, u_##TYPE v) +extern void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v) #endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ @@ -151,17 +129,16 @@ int res = exp; __asm __volatile ( - " " MPLOCKED " " - " cmpxchgl %2,%1 ; " + " " __XSTRING(MPLOCKED) " " + " cmpxchgl %1,%2 ; " " setz %%al ; " " movzbl %%al,%0 ; " "1: " "# atomic_cmpset_int" - : "+a" (res), /* 0 (result) */ - "=m" (*dst) /* 1 */ - : "r" (src), /* 2 */ - "m" (*dst) /* 3 */ - : "memory"); + : "+a" (res) /* 0 (result) */ + : "r" (src), /* 1 */ + "m" (*(dst)) /* 2 */ + : "memory"); return (res); } @@ -172,56 +149,47 @@ long res = exp; __asm __volatile ( - " " MPLOCKED " " - " cmpxchgq %2,%1 ; " + " " __XSTRING(MPLOCKED) " " + " cmpxchgq %1,%2 ; " " setz %%al ; " " movzbq %%al,%0 ; " "1: " "# atomic_cmpset_long" - : "+a" (res), /* 0 (result) */ - "=m" (*dst) /* 1 */ - : "r" (src), /* 2 */ - "m" (*dst) /* 3 */ - : "memory"); + : "+a" (res) /* 0 (result) */ + : "r" (src), /* 1 */ + "m" (*(dst)) /* 2 */ + : "memory"); return (res); } +#endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ -#define ATOMIC_CMPSET(TYPE) \ -static __inline int \ -atomic_cmpset_acq_##TYPE(volatile u_##TYPE *dst, u_##TYPE exp, u_##TYPE src)\ -{ \ - int retval; \ - \ - retval = atomic_cmpset_##TYPE(dst, exp, src); \ - __asm __volatile(MPLFENCE ::: "memory"); \ - return (retval); \ -} \ - \ -static __inline int \ -atomic_cmpset_rel_##TYPE(volatile u_##TYPE *dst, u_##TYPE exp, u_##TYPE src)\ -{ \ - __asm __volatile(MPSFENCE); \ - return (atomic_cmpset_##TYPE(dst, exp, src)); \ -} \ -struct __hack +#if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE) -#define ATOMIC_STORE_LOAD(TYPE) \ +#define ATOMIC_STORE_LOAD(TYPE, LOP, SOP) \ static __inline u_##TYPE \ atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ { \ - u_##TYPE v; \ + u_##TYPE res; \ + \ + __asm __volatile(__XSTRING(MPLOCKED) LOP \ + : "=a" (res), /* 0 (result) */\ + "+m" (*p) /* 1 */ \ + : : "memory"); \ \ - v = *p; \ - __asm __volatile(MPLFENCE ::: "memory"); \ - return (v); \ + return (res); \ } \ \ +/* \ + * The XCHG instruction asserts LOCK automagically. \ + */ \ static __inline void \ atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ { \ - __asm __volatile(MPSFENCE); \ - *p = v; \ + __asm __volatile(SOP \ + : "+m" (*p), /* 0 */ \ + "+r" (v) /* 1 */ \ + : : "memory"); \ } \ struct __hack @@ -230,13 +198,7 @@ extern int atomic_cmpset_int(volatile u_int *, u_int, u_int); extern int atomic_cmpset_long(volatile u_long *, u_long, u_long); -#define ATOMIC_CMPSET(TYPE) \ -extern int atomic_cmpset_acq_##TYPE(volatile u_##TYPE *dst, u_##TYPE exp,\ - u_##TYPE src); \ -extern int atomic_cmpset_rel_##TYPE(volatile u_##TYPE *dst, u_##TYPE exp,\ - u_##TYPE src) - -#define ATOMIC_STORE_LOAD(TYPE) \ +#define ATOMIC_STORE_LOAD(TYPE, LOP, SOP) \ extern u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p); \ extern void atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) @@ -264,17 +226,55 @@ ATOMIC_ASM(add, long, "addq %1,%0", "ir", v); ATOMIC_ASM(subtract, long, "subq %1,%0", "ir", v); -ATOMIC_CMPSET(int); -ATOMIC_CMPSET(long); - -ATOMIC_STORE_LOAD(char); -ATOMIC_STORE_LOAD(short); -ATOMIC_STORE_LOAD(int); -ATOMIC_STORE_LOAD(long); +ATOMIC_STORE_LOAD(char, "cmpxchgb %b0,%1", "xchgb %b1,%0"); +ATOMIC_STORE_LOAD(short,"cmpxchgw %w0,%1", "xchgw %w1,%0"); +ATOMIC_STORE_LOAD(int, "cmpxchgl %0,%1", "xchgl %1,%0"); +ATOMIC_STORE_LOAD(long, "cmpxchgq %0,%1", "xchgq %1,%0"); #undef ATOMIC_ASM #undef ATOMIC_STORE_LOAD +#define atomic_set_acq_char atomic_set_char +#define atomic_set_rel_char atomic_set_char +#define atomic_clear_acq_char atomic_clear_char +#define atomic_clear_rel_char atomic_clear_char +#define atomic_add_acq_char atomic_add_char +#define atomic_add_rel_char atomic_add_char +#define atomic_subtract_acq_char atomic_subtract_char +#define atomic_subtract_rel_char atomic_subtract_char + +#define atomic_set_acq_short atomic_set_short +#define atomic_set_rel_short atomic_set_short +#define atomic_clear_acq_short atomic_clear_short +#define atomic_clear_rel_short atomic_clear_short +#define atomic_add_acq_short atomic_add_short +#define atomic_add_rel_short atomic_add_short +#define atomic_subtract_acq_short atomic_subtract_short +#define atomic_subtract_rel_short atomic_subtract_short + +#define atomic_set_acq_int atomic_set_int +#define atomic_set_rel_int atomic_set_int +#define atomic_clear_acq_int atomic_clear_int +#define atomic_clear_rel_int atomic_clear_int +#define atomic_add_acq_int atomic_add_int +#define atomic_add_rel_int atomic_add_int +#define atomic_subtract_acq_int atomic_subtract_int +#define atomic_subtract_rel_int atomic_subtract_int +#define atomic_cmpset_acq_int atomic_cmpset_int +#define atomic_cmpset_rel_int atomic_cmpset_int + +#define atomic_set_acq_long atomic_set_long +#define atomic_set_rel_long atomic_set_long +#define atomic_clear_acq_long atomic_clear_long +#define atomic_clear_rel_long atomic_clear_long +#define atomic_add_acq_long atomic_add_long +#define atomic_add_rel_long atomic_add_long +#define atomic_subtract_acq_long atomic_subtract_long +#define atomic_subtract_rel_long atomic_subtract_long + +#define atomic_cmpset_acq_ptr atomic_cmpset_ptr +#define atomic_cmpset_rel_ptr atomic_cmpset_ptr + #define atomic_set_8 atomic_set_char #define atomic_set_acq_8 atomic_set_acq_char #define atomic_set_rel_8 atomic_set_rel_char @@ -317,33 +317,63 @@ #define atomic_subtract_32 atomic_subtract_int #define atomic_subtract_acq_32 atomic_subtract_acq_int #define atomic_subtract_rel_32 atomic_subtract_rel_int +#define atomic_load_acq_32 atomic_load_acq_int +#define atomic_store_rel_32 atomic_store_rel_int #define atomic_cmpset_32 atomic_cmpset_int #define atomic_cmpset_acq_32 atomic_cmpset_acq_int #define atomic_cmpset_rel_32 atomic_cmpset_rel_int -#define atomic_load_acq_32 atomic_load_acq_int -#define atomic_store_rel_32 atomic_store_rel_int #define atomic_readandclear_32 atomic_readandclear_int -#define atomic_set_ptr atomic_set_int -#define atomic_set_acq_ptr atomic_set_acq_int -#define atomic_set_rel_ptr atomic_set_rel_int -#define atomic_clear_ptr atomic_clear_int -#define atomic_clear_acq_ptr atomic_clear_acq_int -#define atomic_clear_rel_ptr atomic_clear_rel_int -#define atomic_add_ptr atomic_add_int -#define atomic_add_acq_ptr atomic_add_acq_int -#define atomic_add_rel_ptr atomic_add_rel_int -#define atomic_subtract_ptr atomic_subtract_int -#define atomic_subtract_acq_ptr atomic_subtract_acq_int -#define atomic_subtract_rel_ptr atomic_subtract_rel_int -#define atomic_cmpset_ptr atomic_cmpset_int -#define atomic_cmpset_acq_ptr atomic_cmpset_acq_int -#define atomic_cmpset_rel_ptr atomic_cmpset_rel_int -#define atomic_load_acq_ptr atomic_load_acq_int -#define atomic_store_rel_ptr atomic_store_rel_int -#define atomic_readandclear_ptr atomic_readandclear_int +#if !defined(WANT_FUNCTIONS) +static __inline int +atomic_cmpset_ptr(volatile void *dst, void *exp, void *src) +{ + + return (atomic_cmpset_long((volatile u_long *)dst, + (u_long)exp, (u_long)src)); +} + +static __inline void * +atomic_load_acq_ptr(volatile void *p) +{ + /* + * The apparently-bogus cast to intptr_t in the following is to + * avoid a warning from "gcc -Wbad-function-cast". + */ + return ((void *)(intptr_t)atomic_load_acq_long((volatile u_long *)p)); +} + +static __inline void +atomic_store_rel_ptr(volatile void *p, void *v) +{ + atomic_store_rel_long((volatile u_long *)p, (u_long)v); +} + +#define ATOMIC_PTR(NAME) \ +static __inline void \ +atomic_##NAME##_ptr(volatile void *p, uintptr_t v) \ +{ \ + atomic_##NAME##_long((volatile u_long *)p, v); \ +} \ + \ +static __inline void \ +atomic_##NAME##_acq_ptr(volatile void *p, uintptr_t v) \ +{ \ + atomic_##NAME##_acq_long((volatile u_long *)p, v);\ +} \ + \ +static __inline void \ +atomic_##NAME##_rel_ptr(volatile void *p, uintptr_t v) \ +{ \ + atomic_##NAME##_rel_long((volatile u_long *)p, v);\ +} + +ATOMIC_PTR(set) +ATOMIC_PTR(clear) +ATOMIC_PTR(add) +ATOMIC_PTR(subtract) -#if !defined(WANT_FUNCTIONS) +#undef ATOMIC_PTR #if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE) @@ -356,9 +386,8 @@ " xorl %0,%0 ; " " xchgl %1,%0 ; " "# atomic_readandclear_int" - : "=&r" (result), /* 0 (result) */ - "=m" (*addr) /* 1 (addr) */ - : "m" (*addr)); /* 2 (addr) */ + : "=&r" (result) /* 0 (result) */ + : "m" (*addr)); /* 1 (addr) */ return (result); } @@ -372,9 +401,8 @@ " xorq %0,%0 ; " " xchgq %1,%0 ; " "# atomic_readandclear_int" - : "=&r" (result), /* 0 (result) */ - "=m" (*addr) /* 1 (addr) */ - : "m" (*addr)); /* 2 (addr) */ + : "=&r" (result) /* 0 (result) */ + : "m" (*addr)); /* 1 (addr) */ return (result); } ==== //depot/projects/smpng/sys/amd64/include/bus.h#7 (text+ko) ==== @@ -278,8 +278,8 @@ 1: movb (%2),%%al \n\ stosb \n\ loop 1b" : - "+D" (addr), "+c" (count) : - "r" (bsh + offset) : + "=D" (addr), "=c" (count) : + "r" (bsh + offset), "0" (addr), "1" (count) : "%eax", "memory"); #endif } @@ -299,8 +299,8 @@ 1: movw (%2),%%ax \n\ stosw \n\ loop 1b" : - "+D" (addr), "+c" (count) : - "r" (bsh + offset) : + "=D" (addr), "=c" (count) : + "r" (bsh + offset), "0" (addr), "1" (count) : "%eax", "memory"); #endif } @@ -320,8 +320,8 @@ 1: movl (%2),%%eax \n\ stosl \n\ loop 1b" : - "+D" (addr), "+c" (count) : - "r" (bsh + offset) : + "=D" (addr), "=c" (count) : + "r" (bsh + offset), "0" (addr), "1" (count) : "%eax", "memory"); #endif } @@ -366,7 +366,8 @@ stosb \n\ incl %2 \n\ loop 1b" : - "+D" (addr), "+c" (count), "+d" (_port_) :: + "=D" (addr), "=c" (count), "=d" (_port_) : + "0" (addr), "1" (count), "2" (_port_) : "%eax", "memory", "cc"); #endif } else { @@ -376,7 +377,8 @@ cld \n\ repne \n\ movsb" : - "+D" (addr), "+c" (count), "+S" (_port_) :: + "=D" (addr), "=c" (count), "=S" (_port_) : + "0" (addr), "1" (count), "2" (_port_) : "memory", "cc"); #endif } @@ -396,7 +398,8 @@ stosw \n\ addl $2,%2 \n\ loop 1b" : - "+D" (addr), "+c" (count), "+d" (_port_) :: + "=D" (addr), "=c" (count), "=d" (_port_) : + "0" (addr), "1" (count), "2" (_port_) : "%eax", "memory", "cc"); #endif } else { @@ -406,7 +409,8 @@ cld \n\ repne \n\ movsw" : - "+D" (addr), "+c" (count), "+S" (_port_) :: + "=D" (addr), "=c" (count), "=S" (_port_) : + "0" (addr), "1" (count), "2" (_port_) : "memory", "cc"); #endif } @@ -426,7 +430,8 @@ stosl \n\ addl $4,%2 \n\ loop 1b" : - "+D" (addr), "+c" (count), "+d" (_port_) :: + "=D" (addr), "=c" (count), "=d" (_port_) : + "0" (addr), "1" (count), "2" (_port_) : "%eax", "memory", "cc"); #endif } else { @@ -436,7 +441,8 @@ cld \n\ repne \n\ movsl" : - "+D" (addr), "+c" (count), "+S" (_port_) :: + "=D" (addr), "=c" (count), "=S" (_port_) : + "0" (addr), "1" (count), "2" (_port_) : "memory", "cc"); #endif } @@ -536,8 +542,8 @@ 1: lodsb \n\ movb %%al,(%2) \n\ loop 1b" : - "+S" (addr), "+c" (count) : - "r" (bsh + offset) : + "=S" (addr), "=c" (count) : + "r" (bsh + offset), "0" (addr), "1" (count) : "%eax", "memory", "cc"); #endif } @@ -557,8 +563,8 @@ 1: lodsw \n\ movw %%ax,(%2) \n\ loop 1b" : - "+S" (addr), "+c" (count) : - "r" (bsh + offset) : + "=S" (addr), "=c" (count) : + "r" (bsh + offset), "0" (addr), "1" (count) : "%eax", "memory", "cc"); #endif } @@ -578,8 +584,8 @@ 1: lodsl \n\ movl %%eax,(%2) \n\ loop 1b" : - "+S" (addr), "+c" (count) : - "r" (bsh + offset) : + "=S" (addr), "=c" (count) : + "r" (bsh + offset), "0" (addr), "1" (count) : "%eax", "memory", "cc"); #endif } @@ -625,7 +631,8 @@ outb %%al,%w0 \n\ incl %0 \n\ loop 1b" : - "+d" (_port_), "+S" (addr), "+c" (count) :: + "=d" (_port_), "=S" (addr), "=c" (count) : + "0" (_port_), "1" (addr), "2" (count) : "%eax", "memory", "cc"); #endif } else { @@ -635,7 +642,8 @@ cld \n\ repne \n\ movsb" : - "+D" (_port_), "+S" (addr), "+c" (count) :: + "=D" (_port_), "=S" (addr), "=c" (count) : + "0" (_port_), "1" (addr), "2" (count) : "memory", "cc"); #endif } @@ -655,7 +663,8 @@ outw %%ax,%w0 \n\ addl $2,%0 \n\ loop 1b" : - "+d" (_port_), "+S" (addr), "+c" (count) :: + "=d" (_port_), "=S" (addr), "=c" (count) : + "0" (_port_), "1" (addr), "2" (count) : "%eax", "memory", "cc"); #endif } else { @@ -665,7 +674,8 @@ cld \n\ repne \n\ movsw" : - "+D" (_port_), "+S" (addr), "+c" (count) :: + "=D" (_port_), "=S" (addr), "=c" (count) : + "0" (_port_), "1" (addr), "2" (count) : "memory", "cc"); #endif } @@ -685,7 +695,8 @@ outl %%eax,%w0 \n\ addl $4,%0 \n\ loop 1b" : - "+d" (_port_), "+S" (addr), "+c" (count) :: + "=d" (_port_), "=S" (addr), "=c" (count) : + "0" (_port_), "1" (addr), "2" (count) : "%eax", "memory", "cc"); #endif } else { @@ -695,7 +706,8 @@ cld \n\ repne \n\ movsl" : - "+D" (_port_), "+S" (addr), "+c" (count) :: + "=D" (_port_), "=S" (addr), "=c" (count) : >>> TRUNCATED FOR MAIL (1000 lines) <<<
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200506171955.j5HJtlST050915>