Date: Sun, 8 May 2011 00:39:50 +0000 (UTC) From: Attilio Rao <attilio@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r221614 - projects/largeSMP/sys/powerpc/include Message-ID: <201105080039.p480doiZ021493@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: attilio Date: Sun May 8 00:39:49 2011 New Revision: 221614 URL: http://svn.freebsd.org/changeset/base/221614 Log: All architectures define the size-bounded types (uint32_t, uint64_t, etc.) starting from base C types (int, long, etc). That is also reflected when building atomic operations, as the size-bounded types are built from the base C types. However, powerpc does the inverse thing, leading to a serie of nasty bugs. Cleanup the atomic implementation by defining as base the base C type version and depending on them, appropriately. Tested by: jceel Modified: projects/largeSMP/sys/powerpc/include/atomic.h Modified: projects/largeSMP/sys/powerpc/include/atomic.h ============================================================================== --- projects/largeSMP/sys/powerpc/include/atomic.h Sat May 7 23:34:14 2011 (r221613) +++ projects/largeSMP/sys/powerpc/include/atomic.h Sun May 8 00:39:49 2011 (r221614) @@ -48,13 +48,7 @@ * { *p += v; } */ -#define __ATOMIC_ADD_8(p, v, t) \ - 8-bit atomic_add not implemented - -#define __ATOMIC_ADD_16(p, v, t) \ - 16-bit atomic_add not implemented - -#define __ATOMIC_ADD_32(p, v, t) \ +#define __atomic_add_int(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " add %0, %3, %0\n" \ @@ -63,10 +57,10 @@ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cc", "memory") \ - /* __ATOMIC_ADD_32 */ + /* __atomic_add_int */ #ifdef __powerpc64__ -#define __ATOMIC_ADD_64(p, v, t) \ +#define __atomic_add_long(p, v, t) \ __asm __volatile( \ "1: ldarx %0, 0, %2\n" \ " add %0, %3, %0\n" \ @@ -75,69 +69,78 @@ : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cc", "memory") \ - /* __ATOMIC_ADD_64 */ + /* __atomic_add_long */ #else -#define __ATOMIC_ADD_64(p, v, t) \ - 64-bit atomic_add not implemented +#define __atomic_add_long(p, v, t) \ + long atomic_add not implemented #endif -#define _ATOMIC_ADD(width, suffix, type) \ +#define _ATOMIC_ADD(type) \ static __inline void \ - atomic_add_##suffix(volatile type *p, type v) { \ - type t; \ - __ATOMIC_ADD_##width(p, v, t); \ + atomic_add_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __atomic_add_##type(p, v, t); \ } \ \ static __inline void \ - atomic_add_acq_##suffix(volatile type *p, type v) { \ - type t; \ - __ATOMIC_ADD_##width(p, v, t); \ + atomic_add_acq_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __atomic_add_##type(p, v, t); \ __ATOMIC_BARRIER; \ } \ \ static __inline void \ - atomic_add_rel_##suffix(volatile type *p, type v) { \ - type t; \ + atomic_add_rel_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ __ATOMIC_BARRIER; \ - __ATOMIC_ADD_##width(p, v, t); \ + __atomic_add_##type(p, v, t); \ } \ /* _ATOMIC_ADD */ -#if 0 -_ATOMIC_ADD(8, 8, uint8_t) -_ATOMIC_ADD(8, char, u_char) -_ATOMIC_ADD(16, 16, uint16_t) -_ATOMIC_ADD(16, short, u_short) -#endif -_ATOMIC_ADD(32, 32, uint32_t) -_ATOMIC_ADD(32, int, u_int) -#ifdef __powerpc64__ -_ATOMIC_ADD(64, 64, uint64_t) -_ATOMIC_ADD(64, long, u_long) -_ATOMIC_ADD(64, ptr, uintptr_t) -#else -_ATOMIC_ADD(32, long, u_long) -_ATOMIC_ADD(32, ptr, uintptr_t) -#endif +_ATOMIC_ADD(int) +#define atomic_add_32 atomic_add_int +#define atomic_add_acq_32 atomic_add_acq_int +#define atomic_add_rel_32 atomic_add_rel_int + +#ifdef __powerpc64__ +_ATOMIC_ADD(long) + +#define atomic_add_64 atomic_add_long +#define atomic_add_acq_64 atomic_add_acq_long +#define atomic_add_rel_64 atomic_add_rel_long + +#define atomic_add_ptr(p, v) \ + atomic_add_long((volatile u_long *)(p), (u_long)(v)) +#define atomic_add_acq_ptr(p, v) \ + atomic_add_acq_long((volatile u_long *)(p), (u_long)(v)) +#define atomic_add_rel_ptr(p, v) \ + atomic_add_rel_long((volatile u_long *)(p), (u_long)(v)) +#else +#define atomic_add_long(p, v) \ + atomic_add_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_add_acq_long(p, v) \ + atomic_add_acq_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_add_rel_long(p, v) \ + atomic_add_rel_int((volatile u_int *)(p), (u_int)(v)) + +#define atomic_add_ptr(p, v) \ + atomic_add_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_add_acq_ptr(p, v) \ + atomic_add_acq_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_add_rel_ptr(p, v) \ + atomic_add_rel_int((volatile u_int *)(p), (u_int)(v)) +#endif #undef _ATOMIC_ADD -#undef __ATOMIC_ADD_64 -#undef __ATOMIC_ADD_32 -#undef __ATOMIC_ADD_16 -#undef __ATOMIC_ADD_8 +#undef __atomic_add_long +#undef __atomic_add_int /* * atomic_clear(p, v) * { *p &= ~v; } */ -#define __ATOMIC_CLEAR_8(p, v, t) \ - 8-bit atomic_clear not implemented - -#define __ATOMIC_CLEAR_16(p, v, t) \ - 16-bit atomic_clear not implemented - -#define __ATOMIC_CLEAR_32(p, v, t) \ +#define __atomic_clear_int(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " andc %0, %0, %3\n" \ @@ -146,10 +149,10 @@ _ATOMIC_ADD(32, ptr, uintptr_t) : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cc", "memory") \ - /* __ATOMIC_CLEAR_32 */ + /* __atomic_clear_int */ #ifdef __powerpc64__ -#define __ATOMIC_CLEAR_64(p, v, t) \ +#define __atomic_clear_long(p, v, t) \ __asm __volatile( \ "1: ldarx %0, 0, %2\n" \ " andc %0, %0, %3\n" \ @@ -158,56 +161,72 @@ _ATOMIC_ADD(32, ptr, uintptr_t) : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cc", "memory") \ - /* __ATOMIC_CLEAR_64 */ + /* __atomic_clear_long */ #else -#define __ATOMIC_CLEAR_64(p, v, t) \ - 64-bit atomic_clear not implemented +#define __atomic_clear_long(p, v, t) \ + long atomic_clear not implemented #endif -#define _ATOMIC_CLEAR(width, suffix, type) \ +#define _ATOMIC_CLEAR(type) \ static __inline void \ - atomic_clear_##suffix(volatile type *p, type v) { \ - type t; \ - __ATOMIC_CLEAR_##width(p, v, t); \ + atomic_clear_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __atomic_clear_##type(p, v, t); \ } \ \ static __inline void \ - atomic_clear_acq_##suffix(volatile type *p, type v) { \ - type t; \ - __ATOMIC_CLEAR_##width(p, v, t); \ + atomic_clear_acq_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __atomic_clear_##type(p, v, t); \ __ATOMIC_BARRIER; \ } \ \ static __inline void \ - atomic_clear_rel_##suffix(volatile type *p, type v) { \ - type t; \ + atomic_clear_rel_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ __ATOMIC_BARRIER; \ - __ATOMIC_CLEAR_##width(p, v, t); \ + __atomic_clear_##type(p, v, t); \ } \ /* _ATOMIC_CLEAR */ -#if 0 -_ATOMIC_CLEAR(8, 8, uint8_t) -_ATOMIC_CLEAR(8, char, u_char) -_ATOMIC_CLEAR(16, 16, uint16_t) -_ATOMIC_CLEAR(16, short, u_short) -#endif -_ATOMIC_CLEAR(32, 32, uint32_t) -_ATOMIC_CLEAR(32, int, u_int) -#ifdef __powerpc64__ -_ATOMIC_CLEAR(64, 64, uint64_t) -_ATOMIC_CLEAR(64, long, u_long) -_ATOMIC_CLEAR(64, ptr, uintptr_t) -#else -_ATOMIC_CLEAR(32, long, u_long) -_ATOMIC_CLEAR(32, ptr, uintptr_t) -#endif - -#undef _ATOMIC_CLEAR -#undef __ATOMIC_CLEAR_64 -#undef __ATOMIC_CLEAR_32 -#undef __ATOMIC_CLEAR_16 -#undef __ATOMIC_CLEAR_8 + +_ATOMIC_CLEAR(int) + +#define atomic_clear_32 atomic_clear_int +#define atomic_clear_acq_32 atomic_clear_acq_int +#define atomic_clear_rel_32 atomic_clear_rel_int + +#ifdef __powerpc64__ +_ATOMIC_CLEAR(long) + +#define atomic_clear_64 atomic_clear_long +#define atomic_clear_acq_64 atomic_clear_acq_long +#define atomic_clear_rel_64 atomic_clear_rel_long + +#define atomic_clear_ptr(p, v) \ + atomic_clear_long((volatile u_long *)(p), (u_long)(v)) +#define atomic_clear_acq_ptr(p, v) \ + atomic_clear_acq_long((volatile u_long *)(p), (u_long)(v)) +#define atomic_clear_rel_ptr(p, v) \ + atomic_clear_rel_long((volatile u_long *)(p), (u_long)(v)) +#else +#define atomic_clear_long(p, v) \ + atomic_clear_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_clear_acq_long(p, v) \ + atomic_clear_acq_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_clear_rel_long(p, v) \ + atomic_clear_rel_int((volatile u_int *)(p), (u_int)(v)) + +#define atomic_clear_ptr(p, v) \ + atomic_clear_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_clear_acq_ptr(p, v) \ + atomic_clear_acq_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_clear_rel_ptr(p, v) \ + atomic_clear_rel_int((volatile u_int *)(p), (u_int)(v)) +#endif +#undef _ATOMIC_ADD +#undef __atomic_clear_long +#undef __atomic_clear_int /* * atomic_cmpset(p, o, n) @@ -229,13 +248,7 @@ _ATOMIC_CLEAR(32, ptr, uintptr_t) * { *p |= v; } */ -#define __ATOMIC_SET_8(p, v, t) \ - 8-bit atomic_set not implemented - -#define __ATOMIC_SET_16(p, v, t) \ - 16-bit atomic_set not implemented - -#define __ATOMIC_SET_32(p, v, t) \ +#define __atomic_set_int(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " or %0, %3, %0\n" \ @@ -244,10 +257,10 @@ _ATOMIC_CLEAR(32, ptr, uintptr_t) : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cc", "memory") \ - /* __ATOMIC_SET_32 */ + /* __atomic_set_int */ #ifdef __powerpc64__ -#define __ATOMIC_SET_64(p, v, t) \ +#define __atomic_set_long(p, v, t) \ __asm __volatile( \ "1: ldarx %0, 0, %2\n" \ " or %0, %3, %0\n" \ @@ -256,69 +269,78 @@ _ATOMIC_CLEAR(32, ptr, uintptr_t) : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cc", "memory") \ - /* __ATOMIC_SET_64 */ + /* __atomic_set_long */ #else -#define __ATOMIC_SET_64(p, v, t) \ - 64-bit atomic_set not implemented +#define __atomic_set_long(p, v, t) \ + long atomic_set not implemented #endif -#define _ATOMIC_SET(width, suffix, type) \ +#define _ATOMIC_SET(type) \ static __inline void \ - atomic_set_##suffix(volatile type *p, type v) { \ - type t; \ - __ATOMIC_SET_##width(p, v, t); \ + atomic_set_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __atomic_set_##type(p, v, t); \ } \ \ static __inline void \ - atomic_set_acq_##suffix(volatile type *p, type v) { \ - type t; \ - __ATOMIC_SET_##width(p, v, t); \ + atomic_set_acq_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __atomic_set_##type(p, v, t); \ __ATOMIC_BARRIER; \ } \ \ static __inline void \ - atomic_set_rel_##suffix(volatile type *p, type v) { \ - type t; \ + atomic_set_rel_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ __ATOMIC_BARRIER; \ - __ATOMIC_SET_##width(p, v, t); \ + __atomic_set_##type(p, v, t); \ } \ /* _ATOMIC_SET */ -#if 0 -_ATOMIC_SET(8, 8, uint8_t) -_ATOMIC_SET(8, char, u_char) -_ATOMIC_SET(16, 16, uint16_t) -_ATOMIC_SET(16, short, u_short) -#endif -_ATOMIC_SET(32, 32, uint32_t) -_ATOMIC_SET(32, int, u_int) -#ifdef __powerpc64__ -_ATOMIC_SET(64, 64, uint64_t) -_ATOMIC_SET(64, long, u_long) -_ATOMIC_SET(64, ptr, uintptr_t) -#else -_ATOMIC_SET(32, long, u_long) -_ATOMIC_SET(32, ptr, uintptr_t) -#endif +_ATOMIC_SET(int) +#define atomic_set_32 atomic_set_int +#define atomic_set_acq_32 atomic_set_acq_int +#define atomic_set_rel_32 atomic_set_rel_int + +#ifdef __powerpc64__ +_ATOMIC_SET(long) + +#define atomic_set_64 atomic_set_long +#define atomic_set_acq_64 atomic_set_acq_long +#define atomic_set_rel_64 atomic_set_rel_long + +#define atomic_set_ptr(p, v) \ + atomic_set_long((volatile u_long *)(p), (u_long)(v)) +#define atomic_set_acq_ptr(p, v) \ + atomic_set_acq_long((volatile u_long *)(p), (u_long)(v)) +#define atomic_set_rel_ptr(p, v) \ + atomic_set_rel_long((volatile u_long *)(p), (u_long)(v)) +#else +#define atomic_set_long(p, v) \ + atomic_set_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_set_acq_long(p, v) \ + atomic_set_acq_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_set_rel_long(p, v) \ + atomic_set_rel_int((volatile u_int *)(p), (u_int)(v)) + +#define atomic_set_ptr(p, v) \ + atomic_set_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_set_acq_ptr(p, v) \ + atomic_set_acq_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_set_rel_ptr(p, v) \ + atomic_set_rel_int((volatile u_int *)(p), (u_int)(v)) +#endif #undef _ATOMIC_SET -#undef __ATOMIC_SET_64 -#undef __ATOMIC_SET_32 -#undef __ATOMIC_SET_16 -#undef __ATOMIC_SET_8 +#undef __atomic_set_long +#undef __atomic_set_int /* * atomic_subtract(p, v) * { *p -= v; } */ -#define __ATOMIC_SUBTRACT_8(p, v, t) \ - 8-bit atomic_subtract not implemented - -#define __ATOMIC_SUBTRACT_16(p, v, t) \ - 16-bit atomic_subtract not implemented - -#define __ATOMIC_SUBTRACT_32(p, v, t) \ +#define __atomic_subtract_int(p, v, t) \ __asm __volatile( \ "1: lwarx %0, 0, %2\n" \ " subf %0, %3, %0\n" \ @@ -327,10 +349,10 @@ _ATOMIC_SET(32, ptr, uintptr_t) : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cc", "memory") \ - /* __ATOMIC_SUBTRACT_32 */ + /* __atomic_subtract_int */ #ifdef __powerpc64__ -#define __ATOMIC_SUBTRACT_64(p, v, t) \ +#define __atomic_subtract_long(p, v, t) \ __asm __volatile( \ "1: ldarx %0, 0, %2\n" \ " subf %0, %3, %0\n" \ @@ -339,56 +361,71 @@ _ATOMIC_SET(32, ptr, uintptr_t) : "=&r" (t), "=m" (*p) \ : "r" (p), "r" (v), "m" (*p) \ : "cc", "memory") \ - /* __ATOMIC_SUBTRACT_64 */ + /* __atomic_subtract_long */ #else -#define __ATOMIC_SUBTRACT_64(p, v, t) \ - 64-bit atomic_subtract not implemented +#define __atomic_subtract_long(p, v, t) \ + long atomic_subtract not implemented #endif -#define _ATOMIC_SUBTRACT(width, suffix, type) \ - static __inline void \ - atomic_subtract_##suffix(volatile type *p, type v) { \ - type t; \ - __ATOMIC_SUBTRACT_##width(p, v, t); \ - } \ - \ - static __inline void \ - atomic_subtract_acq_##suffix(volatile type *p, type v) { \ - type t; \ - __ATOMIC_SUBTRACT_##width(p, v, t); \ - __ATOMIC_BARRIER; \ - } \ - \ - static __inline void \ - atomic_subtract_rel_##suffix(volatile type *p, type v) { \ - type t; \ - __ATOMIC_BARRIER; \ - __ATOMIC_SUBTRACT_##width(p, v, t); \ - } \ +#define _ATOMIC_SUBTRACT(type) \ + static __inline void \ + atomic_subtract_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __atomic_subtract_##type(p, v, t); \ + } \ + \ + static __inline void \ + atomic_subtract_acq_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __atomic_subtract_##type(p, v, t); \ + __ATOMIC_BARRIER; \ + } \ + \ + static __inline void \ + atomic_subtract_rel_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __ATOMIC_BARRIER; \ + __atomic_subtract_##type(p, v, t); \ + } \ /* _ATOMIC_SUBTRACT */ -#if 0 -_ATOMIC_SUBTRACT(8, 8, uint8_t) -_ATOMIC_SUBTRACT(8, char, u_char) -_ATOMIC_SUBTRACT(16, 16, uint16_t) -_ATOMIC_SUBTRACT(16, short, u_short) -#endif -_ATOMIC_SUBTRACT(32, 32, uint32_t) -_ATOMIC_SUBTRACT(32, int, u_int) -#ifdef __powerpc64__ -_ATOMIC_SUBTRACT(64, 64, uint64_t) -_ATOMIC_SUBTRACT(64, long, u_long) -_ATOMIC_SUBTRACT(64, ptr, uintptr_t) -#else -_ATOMIC_SUBTRACT(32, long, u_long) -_ATOMIC_SUBTRACT(32, ptr, uintptr_t) -#endif +_ATOMIC_SUBTRACT(int) +#define atomic_subtract_32 atomic_subtract_int +#define atomic_subtract_acq_32 atomic_subtract_acq_int +#define atomic_subtract_rel_32 atomic_subtract_rel_int + +#ifdef __powerpc64__ +_ATOMIC_SUBTRACT(long) + +#define atomic_subtract_64 atomic_subtract_long +#define atomic_subtract_acq_64 atomic_subract_acq_long +#define atomic_subtract_rel_64 atomic_subtract_rel_long + +#define atomic_subtract_ptr(p, v) \ + atomic_subtract_long((volatile u_long *)(p), (u_long)(v)) +#define atomic_subtract_acq_ptr(p, v) \ + atomic_subtract_acq_long((volatile u_long *)(p), (u_long)(v)) +#define atomic_subtract_rel_ptr(p, v) \ + atomic_subtract_rel_long((volatile u_long *)(p), (u_long)(v)) +#else +#define atomic_subtract_long(p, v) \ + atomic_subtract_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_subtract_acq_long(p, v) \ + atomic_subtract_acq_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_subtract_rel_long(p, v) \ + atomic_subtract_rel_int((volatile u_int *)(p), (u_int)(v)) + +#define atomic_subtract_ptr(p, v) \ + atomic_subtract_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_subtract_acq_ptr(p, v) \ + atomic_subtract_acq_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_subtract_rel_ptr(p, v) \ + atomic_subtract_rel_int((volatile u_int *)(p), (u_int)(v)) +#endif #undef _ATOMIC_SUBTRACT -#undef __ATOMIC_SUBTRACT_64 -#undef __ATOMIC_SUBTRACT_32 -#undef __ATOMIC_SUBTRACT_16 -#undef __ATOMIC_SUBTRACT_8 +#undef __atomic_subtract_long +#undef __atomic_subtract_int /* * atomic_store_rel(p, v) @@ -399,10 +436,10 @@ _ATOMIC_SUBTRACT(32, ptr, uintptr_t) * Old/original implementations that still need revisiting. */ -static __inline uint32_t -atomic_readandclear_32(volatile uint32_t *addr) +static __inline u_int +atomic_readandclear_int(volatile u_int *addr) { - uint32_t result,temp; + u_int result,temp; #ifdef __GNUCLIKE_ASM __asm __volatile ( @@ -420,10 +457,10 @@ atomic_readandclear_32(volatile uint32_t } #ifdef __powerpc64__ -static __inline uint64_t -atomic_readandclear_64(volatile uint64_t *addr) +static __inline u_long +atomic_readandclear_long(volatile u_long *addr) { - uint64_t result,temp; + u_long result,temp; #ifdef __GNUCLIKE_ASM __asm __volatile ( @@ -441,37 +478,25 @@ atomic_readandclear_64(volatile uint64_t } #endif -#define atomic_readandclear_int atomic_readandclear_32 +#define atomic_readandclear_32 atomic_readandclear_int #ifdef __powerpc64__ -#define atomic_readandclear_long atomic_readandclear_64 -#define atomic_readandclear_ptr atomic_readandclear_64 +#define atomic_readandclear_64 atomic_readandclear_long + +#define atomic_readandclear_ptr(p) \ + atomic_readandclear_long((volatile u_long *)(p)) #else -#define atomic_readandclear_long atomic_readandclear_32 -#define atomic_readandclear_ptr atomic_readandclear_32 +#define atomic_readandclear_long(p) \ + atomic_readandclear_int((volatile u_int *)(p)) + +#define atomic_readandclear_ptr(p) \ + atomic_readandclear_int((volatile u_int *)(p)) #endif /* * We assume that a = b will do atomic loads and stores. */ -#define ATOMIC_STORE_LOAD(TYPE, WIDTH) \ -static __inline u_##TYPE \ -atomic_load_acq_##WIDTH(volatile u_##TYPE *p) \ -{ \ - u_##TYPE v; \ - \ - v = *p; \ - __ATOMIC_BARRIER; \ - return (v); \ -} \ - \ -static __inline void \ -atomic_store_rel_##WIDTH(volatile u_##TYPE *p, u_##TYPE v) \ -{ \ - __ATOMIC_BARRIER; \ - *p = v; \ -} \ - \ +#define ATOMIC_STORE_LOAD(TYPE) \ static __inline u_##TYPE \ atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ { \ @@ -489,25 +514,32 @@ atomic_store_rel_##TYPE(volatile u_##TYP *p = v; \ } -ATOMIC_STORE_LOAD(char, 8) -ATOMIC_STORE_LOAD(short, 16) -ATOMIC_STORE_LOAD(int, 32) -#ifdef __powerpc64__ -ATOMIC_STORE_LOAD(long, 64) -#endif +ATOMIC_STORE_LOAD(int) + +#define atomic_load_acq_32 atomic_load_acq_int +#define atomic_store_rel_32 atomic_store_rel_int #ifdef __powerpc64__ -#define atomic_load_acq_long atomic_load_acq_64 -#define atomic_store_rel_long atomic_store_rel_64 -#define atomic_load_acq_ptr atomic_load_acq_64 -#define atomic_store_rel_ptr atomic_store_rel_64 +ATOMIC_STORE_LOAD(long) + +#define atomic_load_acq_64 atomic_load_acq_long +#define atomic_store_rel_64 atomic_store_rel_long + +#define atomic_load_acq_ptr(p) \ + atomic_load_acq_long((volatile u_long *)(p)) +#define atomic_store_rel_ptr(p, v) \ + atomic_store_rel_long((volatile u_long *)(p), (u_long)(v)) #else -#define atomic_load_acq_long atomic_load_acq_32 -#define atomic_store_rel_long atomic_store_rel_32 -#define atomic_load_acq_ptr atomic_load_acq_32 -#define atomic_store_rel_ptr atomic_store_rel_32 -#endif +#define atomic_load_acq_long(p) \ + atomic_load_acq_int((volatile u_int *)(p)) +#define atomic_store_rel_long(p, v) \ + atomic_store_rel_int((volatile u_int *)(p), (u_int)(v)) +#define atomic_load_acq_ptr(p) \ + atomic_load_acq_int((volatile u_int *)(p)) +#define atomic_store_rel_ptr(p, v) \ + atomic_store_rel_int((volatile u_int *)(p), (u_int)(v)) +#endif #undef ATOMIC_STORE_LOAD /* @@ -516,7 +548,7 @@ ATOMIC_STORE_LOAD(long, 64) * zero if the compare failed, nonzero otherwise. */ static __inline int -atomic_cmpset_32(volatile uint32_t* p, uint32_t cmpval, uint32_t newval) +atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval) { int ret; @@ -542,32 +574,43 @@ atomic_cmpset_32(volatile uint32_t* p, u } static __inline int +atomic_cmpset_acq_int(volatile u_int *p, u_int cmpval, u_int newval) +{ + int retval; + + retval = atomic_cmpset_int(p, cmpval, newval); + __ATOMIC_BARRIER; + return (retval); +} + +static __inline int +atomic_cmpset_rel_int(volatile u_int *p, u_int cmpval, u_int newval) +{ + __ATOMIC_BARRIER; + return (atomic_cmpset_int(p, cmpval, newval)); +} + +#define atomic_cmpset_32 atomic_cmpset_int +#define atomic_cmpset_acq_32 atomic_cmpset_acq_int +#define atomic_cmpset_rel_32 atomic_cmpset_rel_int + +#ifdef __powerpc64__ +static __inline int atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval) { int ret; #ifdef __GNUCLIKE_ASM __asm __volatile ( - #ifdef __powerpc64__ "1:\tldarx %0, 0, %2\n\t" /* load old value */ "cmpld %3, %0\n\t" /* compare */ "bne 2f\n\t" /* exit if not equal */ "stdcx. %4, 0, %2\n\t" /* attempt to store */ - #else - "1:\tlwarx %0, 0, %2\n\t" /* load old value */ - "cmplw %3, %0\n\t" /* compare */ - "bne 2f\n\t" /* exit if not equal */ - "stwcx. %4, 0, %2\n\t" /* attempt to store */ - #endif "bne- 1b\n\t" /* spin if failed */ "li %0, 1\n\t" /* success - retval = 1 */ "b 3f\n\t" /* we've succeeded */ "2:\n\t" - #ifdef __powerpc64__ "stdcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ - #else - "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ - #endif "li %0, 0\n\t" /* failure - retval = 0 */ "3:\n\t" : "=&r" (ret), "=m" (*p) @@ -578,33 +621,6 @@ atomic_cmpset_long(volatile u_long* p, u return (ret); } -#define atomic_cmpset_int atomic_cmpset_32 - -#ifdef __powerpc64__ -#define atomic_cmpset_ptr(dst, old, new) \ - atomic_cmpset_long((volatile u_long *)(dst), (u_long)(old), (u_long)(new)) -#else -#define atomic_cmpset_ptr(dst, old, new) \ - atomic_cmpset_32((volatile u_int *)(dst), (u_int)(old), (u_int)(new)) -#endif - -static __inline int -atomic_cmpset_acq_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) -{ - int retval; - - retval = atomic_cmpset_32(p, cmpval, newval); - __ATOMIC_BARRIER; - return (retval); -} - -static __inline int -atomic_cmpset_rel_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) -{ - __ATOMIC_BARRIER; - return (atomic_cmpset_32(p, cmpval, newval)); -} - static __inline int atomic_cmpset_acq_long(volatile u_long *p, u_long cmpval, u_long newval) { @@ -622,39 +638,59 @@ atomic_cmpset_rel_long(volatile u_long * return (atomic_cmpset_long(p, cmpval, newval)); } -#define atomic_cmpset_acq_int atomic_cmpset_acq_32 -#define atomic_cmpset_rel_int atomic_cmpset_rel_32 - -#ifdef __powerpc64__ -#define atomic_cmpset_acq_ptr(dst, old, new) \ - atomic_cmpset_acq_long((volatile u_long *)(dst), (u_long)(old), (u_long)(new)) -#define atomic_cmpset_rel_ptr(dst, old, new) \ - atomic_cmpset_rel_long((volatile u_long *)(dst), (u_long)(old), (u_long)(new)) -#else -#define atomic_cmpset_acq_ptr(dst, old, new) \ - atomic_cmpset_acq_32((volatile u_int *)(dst), (u_int)(old), (u_int)(new)) -#define atomic_cmpset_rel_ptr(dst, old, new) \ - atomic_cmpset_rel_32((volatile u_int *)(dst), (u_int)(old), (u_int)(new)) +#define atomic_cmpset_64 atomic_cmpset_long +#define atomic_cmpset_acq_64 atomic_cmpset_acq_long +#define atomic_cmpset_rel_64 atomic_cmpset_rel_long + +#define atomic_cmpset_ptr(dst, old, new) \ + atomic_cmpset_long((volatile u_long *)(dst), (u_long)(old), \ + (u_long)(new)) +#define atomic_cmpset_acq_ptr(dst, old, new) \ + atomic_cmpset_acq_long((volatile u_long *)(dst), (u_long)(old), \ + (u_long)(new)) +#define atomic_cmpset_rel_ptr(dst, old, new) \ + atomic_cmpset_rel_long((volatile u_long *)(dst), (u_long)(old), \ + (u_long)(new)) +#else +#define atomic_cmpset_long(dst, old, new) \ + atomic_cmpset_int((volatile u_int *)(dst), (u_int)(old), \ + (u_int)(new)) +#define atomic_cmpset_acq_long(dst, old, new) \ + atomic_cmpset_acq_int((volatile u_int *)(dst), (u_int)(old), \ + (u_int)(new)) +#define atomic_cmpset_rel_long(dst, old, new) \ + atomic_cmpset_rel_int((volatile u_int *)(dst), (u_int)(old), \ + (u_int)(new)) + +#define atomic_cmpset_ptr(dst, old, new) \ + atomic_cmpset_int((volatile u_int *)(dst), (u_int)(old), \ + (u_int)(new)) +#define atomic_cmpset_acq_ptr(dst, old, new) \ + atomic_cmpset_acq_int((volatile u_int *)(dst), (u_int)(old), \ + (u_int)(new)) +#define atomic_cmpset_rel_ptr(dst, old, new) \ + atomic_cmpset_rel_int((volatile u_int *)(dst), (u_int)(old), \ + (u_int)(new)) #endif -static __inline uint32_t -atomic_fetchadd_32(volatile uint32_t *p, uint32_t v) +static __inline u_int +atomic_fetchadd_int(volatile u_int *p, u_int v) { - uint32_t value; + u_int value; do { value = *p; - } while (!atomic_cmpset_32(p, value, value + v)); + } while (!atomic_cmpset_int(p, value, value + v)); return (value); } -#define atomic_fetchadd_int atomic_fetchadd_32 +#define atomic_fetchadd_32 atomic_fetchadd_int #ifdef __powerpc64__ -static __inline uint64_t -atomic_fetchadd_64(volatile uint64_t *p, uint64_t v) +static __inline u_long +atomic_fetchadd_long(volatile u_long *p, u_long v) { - uint64_t value; + u_long value; do { value = *p; @@ -662,10 +698,10 @@ atomic_fetchadd_64(volatile uint64_t *p, return (value); } -#define atomic_fetchadd_long atomic_fetchadd_64 +#define atomic_fetchadd_64 atomic_fetchadd_long #else -#define atomic_fetchadd_long(p, v) \ - (u_long)atomic_fetchadd_32((volatile u_int *)(p), (u_int)(v)) +#define atomic_fetchadd_long(p, v) \ + (u_long)atomic_fetchadd_int((volatile u_int *)(p), (u_int)(v)) #endif #endif /* ! _MACHINE_ATOMIC_H_ */
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201105080039.p480doiZ021493>