mirror of
https://github.com/revyos/thead-kernel.git
synced 2026-09-06 09:54:35 +02:00
556f057aca
After selecting ARCH_USE_CMPXCHG_LOCKREF, one straight futher optimization is implementing the arch_cmpxchg64_relaxed() because the lockref code does not need the cmpxchg to have barrier semantics. At the same time, implement arch_cmpxchg64_acquire and arch_cmpxchg64_release as well. However, on both TH1520 and JH7110 platforms, I didn't see obvious performance improvement with Linus' test case [1]. IMHO, this may be related with the fence and lr.d/sc.d hw implementations. In theory, lr/sc without fence could give performance improvement over lr/sc plus fence, so add the code here to leave performance improvement room on newer HW platforms. Link: http://marc.info/?l=linux-fsdevel&m=137782380714721&w=4 [1] Signed-off-by: Jisheng Zhang <jszhang@kernel.org>
418 lines
11 KiB
C
418 lines
11 KiB
C
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
/*
|
|
* Copyright (C) 2014 Regents of the University of California
|
|
*/
|
|
|
|
#ifndef _ASM_RISCV_CMPXCHG_H
|
|
#define _ASM_RISCV_CMPXCHG_H
|
|
|
|
#include <linux/bug.h>
|
|
|
|
#include <asm/barrier.h>
|
|
#include <asm/fence.h>
|
|
|
|
static inline ulong __xchg16_relaxed(ulong new, void *ptr)
|
|
{
|
|
ulong ret, tmp;
|
|
ulong shif = ((ulong)ptr & 2) ? 16 : 0;
|
|
ulong mask = 0xffff << shif;
|
|
ulong *__ptr = (ulong *)((ulong)ptr & ~2);
|
|
|
|
__asm__ __volatile__ (
|
|
"0: lr.w %0, %2\n"
|
|
" and %1, %0, %z3\n"
|
|
" or %1, %1, %z4\n"
|
|
" sc.w %1, %1, %2\n"
|
|
" bnez %1, 0b\n"
|
|
: "=&r" (ret), "=&r" (tmp), "+A" (*__ptr)
|
|
: "rJ" (~mask), "rJ" (new << shif)
|
|
: "memory");
|
|
|
|
return (ulong)((ret & mask) >> shif);
|
|
}
|
|
|
|
#define __xchg_relaxed(ptr, new, size) \
|
|
({ \
|
|
__typeof__(ptr) __ptr = (ptr); \
|
|
__typeof__(new) __new = (new); \
|
|
__typeof__(*(ptr)) __ret; \
|
|
switch (size) { \
|
|
case 2: { \
|
|
__ret = (__typeof__(*(ptr))) \
|
|
__xchg16_relaxed((ulong)__new, __ptr); \
|
|
break;} \
|
|
case 4: \
|
|
__asm__ __volatile__ ( \
|
|
" amoswap.w %0, %2, %1\n" \
|
|
: "=r" (__ret), "+A" (*__ptr) \
|
|
: "r" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
case 8: \
|
|
__asm__ __volatile__ ( \
|
|
" amoswap.d %0, %2, %1\n" \
|
|
: "=r" (__ret), "+A" (*__ptr) \
|
|
: "r" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
default: \
|
|
BUILD_BUG(); \
|
|
} \
|
|
__ret; \
|
|
})
|
|
|
|
#define xchg_relaxed(ptr, x) \
|
|
({ \
|
|
__typeof__(*(ptr)) _x_ = (x); \
|
|
(__typeof__(*(ptr))) __xchg_relaxed((ptr), \
|
|
_x_, sizeof(*(ptr))); \
|
|
})
|
|
|
|
#define __xchg_acquire(ptr, new, size) \
|
|
({ \
|
|
__typeof__(ptr) __ptr = (ptr); \
|
|
__typeof__(new) __new = (new); \
|
|
__typeof__(*(ptr)) __ret; \
|
|
switch (size) { \
|
|
case 4: \
|
|
__asm__ __volatile__ ( \
|
|
" amoswap.w %0, %2, %1\n" \
|
|
RISCV_ACQUIRE_BARRIER \
|
|
: "=r" (__ret), "+A" (*__ptr) \
|
|
: "r" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
case 8: \
|
|
__asm__ __volatile__ ( \
|
|
" amoswap.d %0, %2, %1\n" \
|
|
RISCV_ACQUIRE_BARRIER \
|
|
: "=r" (__ret), "+A" (*__ptr) \
|
|
: "r" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
default: \
|
|
BUILD_BUG(); \
|
|
} \
|
|
__ret; \
|
|
})
|
|
|
|
#define xchg_acquire(ptr, x) \
|
|
({ \
|
|
__typeof__(*(ptr)) _x_ = (x); \
|
|
(__typeof__(*(ptr))) __xchg_acquire((ptr), \
|
|
_x_, sizeof(*(ptr))); \
|
|
})
|
|
|
|
#define __xchg_release(ptr, new, size) \
|
|
({ \
|
|
__typeof__(ptr) __ptr = (ptr); \
|
|
__typeof__(new) __new = (new); \
|
|
__typeof__(*(ptr)) __ret; \
|
|
switch (size) { \
|
|
case 4: \
|
|
__asm__ __volatile__ ( \
|
|
RISCV_RELEASE_BARRIER \
|
|
" amoswap.w %0, %2, %1\n" \
|
|
: "=r" (__ret), "+A" (*__ptr) \
|
|
: "r" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
case 8: \
|
|
__asm__ __volatile__ ( \
|
|
RISCV_RELEASE_BARRIER \
|
|
" amoswap.d %0, %2, %1\n" \
|
|
: "=r" (__ret), "+A" (*__ptr) \
|
|
: "r" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
default: \
|
|
BUILD_BUG(); \
|
|
} \
|
|
__ret; \
|
|
})
|
|
|
|
#define xchg_release(ptr, x) \
|
|
({ \
|
|
__typeof__(*(ptr)) _x_ = (x); \
|
|
(__typeof__(*(ptr))) __xchg_release((ptr), \
|
|
_x_, sizeof(*(ptr))); \
|
|
})
|
|
|
|
#define __xchg(ptr, new, size) \
|
|
({ \
|
|
__typeof__(ptr) __ptr = (ptr); \
|
|
__typeof__(new) __new = (new); \
|
|
__typeof__(*(ptr)) __ret; \
|
|
switch (size) { \
|
|
case 4: \
|
|
__asm__ __volatile__ ( \
|
|
" amoswap.w.aqrl %0, %2, %1\n" \
|
|
: "=r" (__ret), "+A" (*__ptr) \
|
|
: "r" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
case 8: \
|
|
__asm__ __volatile__ ( \
|
|
" amoswap.d.aqrl %0, %2, %1\n" \
|
|
: "=r" (__ret), "+A" (*__ptr) \
|
|
: "r" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
default: \
|
|
BUILD_BUG(); \
|
|
} \
|
|
__ret; \
|
|
})
|
|
|
|
#define xchg(ptr, x) \
|
|
({ \
|
|
__typeof__(*(ptr)) _x_ = (x); \
|
|
(__typeof__(*(ptr))) __xchg((ptr), _x_, sizeof(*(ptr))); \
|
|
})
|
|
|
|
#define xchg32(ptr, x) \
|
|
({ \
|
|
BUILD_BUG_ON(sizeof(*(ptr)) != 4); \
|
|
xchg((ptr), (x)); \
|
|
})
|
|
|
|
#define xchg64(ptr, x) \
|
|
({ \
|
|
BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
|
|
xchg((ptr), (x)); \
|
|
})
|
|
|
|
/*
|
|
* Atomic compare and exchange. Compare OLD with MEM, if identical,
|
|
* store NEW in MEM. Return the initial value in MEM. Success is
|
|
* indicated by comparing RETURN with OLD.
|
|
*/
|
|
#define __cmpxchg_relaxed(ptr, old, new, size) \
|
|
({ \
|
|
__typeof__(ptr) __ptr = (ptr); \
|
|
__typeof__(*(ptr)) __old = (old); \
|
|
__typeof__(*(ptr)) __new = (new); \
|
|
__typeof__(*(ptr)) __ret; \
|
|
register unsigned int __rc; \
|
|
switch (size) { \
|
|
case 4: \
|
|
__asm__ __volatile__ ( \
|
|
"0: lr.w %0, %2\n" \
|
|
" bne %0, %z3, 1f\n" \
|
|
" sc.w %1, %z4, %2\n" \
|
|
" bnez %1, 0b\n" \
|
|
"1:\n" \
|
|
: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
|
|
: "rJ" ((long)__old), "rJ" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
case 8: \
|
|
__asm__ __volatile__ ( \
|
|
"0: lr.d %0, %2\n" \
|
|
" bne %0, %z3, 1f\n" \
|
|
" sc.d %1, %z4, %2\n" \
|
|
" bnez %1, 0b\n" \
|
|
"1:\n" \
|
|
: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
|
|
: "rJ" (__old), "rJ" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
default: \
|
|
BUILD_BUG(); \
|
|
} \
|
|
__ret; \
|
|
})
|
|
|
|
#define cmpxchg_relaxed(ptr, o, n) \
|
|
({ \
|
|
__typeof__(*(ptr)) _o_ = (o); \
|
|
__typeof__(*(ptr)) _n_ = (n); \
|
|
(__typeof__(*(ptr))) __cmpxchg_relaxed((ptr), \
|
|
_o_, _n_, sizeof(*(ptr))); \
|
|
})
|
|
|
|
#define __cmpxchg_acquire(ptr, old, new, size) \
|
|
({ \
|
|
__typeof__(ptr) __ptr = (ptr); \
|
|
__typeof__(*(ptr)) __old = (old); \
|
|
__typeof__(*(ptr)) __new = (new); \
|
|
__typeof__(*(ptr)) __ret; \
|
|
register unsigned int __rc; \
|
|
switch (size) { \
|
|
case 4: \
|
|
__asm__ __volatile__ ( \
|
|
"0: lr.w %0, %2\n" \
|
|
" bne %0, %z3, 1f\n" \
|
|
" sc.w %1, %z4, %2\n" \
|
|
" bnez %1, 0b\n" \
|
|
RISCV_ACQUIRE_BARRIER \
|
|
"1:\n" \
|
|
: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
|
|
: "rJ" ((long)__old), "rJ" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
case 8: \
|
|
__asm__ __volatile__ ( \
|
|
"0: lr.d %0, %2\n" \
|
|
" bne %0, %z3, 1f\n" \
|
|
" sc.d %1, %z4, %2\n" \
|
|
" bnez %1, 0b\n" \
|
|
RISCV_ACQUIRE_BARRIER \
|
|
"1:\n" \
|
|
: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
|
|
: "rJ" (__old), "rJ" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
default: \
|
|
BUILD_BUG(); \
|
|
} \
|
|
__ret; \
|
|
})
|
|
|
|
#define cmpxchg_acquire(ptr, o, n) \
|
|
({ \
|
|
__typeof__(*(ptr)) _o_ = (o); \
|
|
__typeof__(*(ptr)) _n_ = (n); \
|
|
(__typeof__(*(ptr))) __cmpxchg_acquire((ptr), \
|
|
_o_, _n_, sizeof(*(ptr))); \
|
|
})
|
|
|
|
#define __cmpxchg_release(ptr, old, new, size) \
|
|
({ \
|
|
__typeof__(ptr) __ptr = (ptr); \
|
|
__typeof__(*(ptr)) __old = (old); \
|
|
__typeof__(*(ptr)) __new = (new); \
|
|
__typeof__(*(ptr)) __ret; \
|
|
register unsigned int __rc; \
|
|
switch (size) { \
|
|
case 4: \
|
|
__asm__ __volatile__ ( \
|
|
RISCV_RELEASE_BARRIER \
|
|
"0: lr.w %0, %2\n" \
|
|
" bne %0, %z3, 1f\n" \
|
|
" sc.w %1, %z4, %2\n" \
|
|
" bnez %1, 0b\n" \
|
|
"1:\n" \
|
|
: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
|
|
: "rJ" ((long)__old), "rJ" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
case 8: \
|
|
__asm__ __volatile__ ( \
|
|
RISCV_RELEASE_BARRIER \
|
|
"0: lr.d %0, %2\n" \
|
|
" bne %0, %z3, 1f\n" \
|
|
" sc.d %1, %z4, %2\n" \
|
|
" bnez %1, 0b\n" \
|
|
"1:\n" \
|
|
: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
|
|
: "rJ" (__old), "rJ" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
default: \
|
|
BUILD_BUG(); \
|
|
} \
|
|
__ret; \
|
|
})
|
|
|
|
#define cmpxchg_release(ptr, o, n) \
|
|
({ \
|
|
__typeof__(*(ptr)) _o_ = (o); \
|
|
__typeof__(*(ptr)) _n_ = (n); \
|
|
(__typeof__(*(ptr))) __cmpxchg_release((ptr), \
|
|
_o_, _n_, sizeof(*(ptr))); \
|
|
})
|
|
|
|
#define __cmpxchg(ptr, old, new, size) \
|
|
({ \
|
|
__typeof__(ptr) __ptr = (ptr); \
|
|
__typeof__(*(ptr)) __old = (old); \
|
|
__typeof__(*(ptr)) __new = (new); \
|
|
__typeof__(*(ptr)) __ret; \
|
|
register unsigned int __rc; \
|
|
switch (size) { \
|
|
case 4: \
|
|
__asm__ __volatile__ ( \
|
|
"0: lr.w %0, %2\n" \
|
|
" bne %0, %z3, 1f\n" \
|
|
" sc.w.rl %1, %z4, %2\n" \
|
|
" bnez %1, 0b\n" \
|
|
" fence rw, rw\n" \
|
|
"1:\n" \
|
|
: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
|
|
: "rJ" ((long)__old), "rJ" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
case 8: \
|
|
__asm__ __volatile__ ( \
|
|
"0: lr.d %0, %2\n" \
|
|
" bne %0, %z3, 1f\n" \
|
|
" sc.d.rl %1, %z4, %2\n" \
|
|
" bnez %1, 0b\n" \
|
|
" fence rw, rw\n" \
|
|
"1:\n" \
|
|
: "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \
|
|
: "rJ" (__old), "rJ" (__new) \
|
|
: "memory"); \
|
|
break; \
|
|
default: \
|
|
BUILD_BUG(); \
|
|
} \
|
|
__ret; \
|
|
})
|
|
|
|
#define cmpxchg(ptr, o, n) \
|
|
({ \
|
|
__typeof__(*(ptr)) _o_ = (o); \
|
|
__typeof__(*(ptr)) _n_ = (n); \
|
|
(__typeof__(*(ptr))) __cmpxchg((ptr), \
|
|
_o_, _n_, sizeof(*(ptr))); \
|
|
})
|
|
|
|
#define cmpxchg_local(ptr, o, n) \
|
|
(__cmpxchg_relaxed((ptr), (o), (n), sizeof(*(ptr))))
|
|
|
|
#define cmpxchg32(ptr, o, n) \
|
|
({ \
|
|
BUILD_BUG_ON(sizeof(*(ptr)) != 4); \
|
|
cmpxchg((ptr), (o), (n)); \
|
|
})
|
|
|
|
#define cmpxchg32_local(ptr, o, n) \
|
|
({ \
|
|
BUILD_BUG_ON(sizeof(*(ptr)) != 4); \
|
|
cmpxchg_relaxed((ptr), (o), (n)) \
|
|
})
|
|
|
|
#define cmpxchg64(ptr, o, n) \
|
|
({ \
|
|
BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
|
|
cmpxchg((ptr), (o), (n)); \
|
|
})
|
|
|
|
#define cmpxchg64_local(ptr, o, n) \
|
|
({ \
|
|
BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
|
|
cmpxchg_relaxed((ptr), (o), (n)); \
|
|
})
|
|
|
|
#define cmpxchg64_relaxed(ptr, o, n) \
|
|
({ \
|
|
BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
|
|
cmpxchg_relaxed((ptr), (o), (n)); \
|
|
})
|
|
|
|
#define cmpxchg64_acquire(ptr, o, n) \
|
|
({ \
|
|
BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
|
|
cmpxchg_acquire((ptr), (o), (n)); \
|
|
})
|
|
|
|
#define cmpxchg64_release(ptr, o, n) \
|
|
({ \
|
|
BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
|
|
cmpxchg_release((ptr), (o), (n)); \
|
|
})
|
|
|
|
#endif /* _ASM_RISCV_CMPXCHG_H */
|