powerpc: Use lwarx/ldarx hint in bit locks

This patch implements the lwarx/ldarx hint bit for bit locks.

Signed-off-by: Anton Blanchard <anton@samba.org>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>

diff --git a/arch/powerpc/include/asm/asm-compat.h b/arch/powerpc/include/asm/asm-compat.h
index 8f0fe797194992829dd5926ef2d80605415e5059..c1b475a941eb9d08a272f6b378cfdfa7d14cdf9c 100644 (file)
--- a/arch/powerpc/include/asm/asm-compat.h
+++ b/arch/powerpc/include/asm/asm-compat.h
@@ -2,6 +2,7 @@
 #define _ASM_POWERPC_ASM_COMPAT_H
 
 #include <asm/types.h>
+#include <asm/ppc-opcode.h>
 
 #ifdef __ASSEMBLY__
 #  define stringify_in_c(...)  __VA_ARGS__
@@ -24,7 +25,7 @@
 #define PPC_LONG       stringify_in_c(.llong)
 #define PPC_LONG_ALIGN stringify_in_c(.balign 8)
 #define PPC_TLNEI      stringify_in_c(tdnei)
-#define PPC_LLARX      stringify_in_c(ldarx)
+#define PPC_LLARX(t, a, b, eh) PPC_LDARX(t, a, b, eh)
 #define PPC_STLCX      stringify_in_c(stdcx.)
 #define PPC_CNTLZL     stringify_in_c(cntlzd)
 
@@ -46,7 +47,7 @@
 #define PPC_LONG       stringify_in_c(.long)
 #define PPC_LONG_ALIGN stringify_in_c(.balign 4)
 #define PPC_TLNEI      stringify_in_c(twnei)
-#define PPC_LLARX      stringify_in_c(lwarx)
+#define PPC_LLARX(t, a, b, eh) PPC_LWARX(t, a, b, eh)
 #define PPC_STLCX      stringify_in_c(stwcx.)
 #define PPC_CNTLZL     stringify_in_c(cntlzw)
 #define PPC_MTOCRF     stringify_in_c(mtcrf)

diff --git a/arch/powerpc/include/asm/bitops.h b/arch/powerpc/include/asm/bitops.h
index 56f2f2ea56319fc13897e5a94647ae54f6fc5cd3..3c7c37bd92e38baf590561b2dae7b4286e065976 100644 (file)
--- a/arch/powerpc/include/asm/bitops.h
+++ b/arch/powerpc/include/asm/bitops.h
@@ -65,7 +65,7 @@ static __inline__ void fn(unsigned long mask, \
        unsigned long *p = (unsigned long *)_p; \
        __asm__ __volatile__ (                  \
        prefix                                  \
-"1:"   PPC_LLARX "%0,0,%3\n"                   \
+"1:"   PPC_LLARX(%0,0,%3,0) "\n"               \
        stringify_in_c(op) "%0,%0,%2\n"         \
        PPC405_ERR77(0,%3)                      \
        PPC_STLCX "%0,0,%3\n"                   \
@@ -103,31 +103,31 @@ static __inline__ void change_bit(int nr, volatile unsigned long *addr)
 
 /* Like DEFINE_BITOP(), with changes to the arguments to 'op' and the output
  * operands. */
-#define DEFINE_TESTOP(fn, op, prefix, postfix) \
-static __inline__ unsigned long fn(            \
-               unsigned long mask,             \
-               volatile unsigned long *_p)     \
-{                                              \
-       unsigned long old, t;                   \
-       unsigned long *p = (unsigned long *)_p; \
-       __asm__ __volatile__ (                  \
-       prefix                                  \
-"1:"   PPC_LLARX "%0,0,%3\n"                   \
-       stringify_in_c(op) "%1,%0,%2\n"         \
-       PPC405_ERR77(0,%3)                      \
-       PPC_STLCX "%1,0,%3\n"                   \
-       "bne- 1b\n"                             \
-       postfix                                 \
-       : "=&r" (old), "=&r" (t)                \
-       : "r" (mask), "r" (p)                   \
-       : "cc", "memory");                      \
-       return (old & mask);                    \
+#define DEFINE_TESTOP(fn, op, prefix, postfix, eh)     \
+static __inline__ unsigned long fn(                    \
+               unsigned long mask,                     \
+               volatile unsigned long *_p)             \
+{                                                      \
+       unsigned long old, t;                           \
+       unsigned long *p = (unsigned long *)_p;         \
+       __asm__ __volatile__ (                          \
+       prefix                                          \
+"1:"   PPC_LLARX(%0,0,%3,eh) "\n"                      \
+       stringify_in_c(op) "%1,%0,%2\n"                 \
+       PPC405_ERR77(0,%3)                              \
+       PPC_STLCX "%1,0,%3\n"                           \
+       "bne- 1b\n"                                     \
+       postfix                                         \
+       : "=&r" (old), "=&r" (t)                        \
+       : "r" (mask), "r" (p)                           \
+       : "cc", "memory");                              \
+       return (old & mask);                            \
 }
 
-DEFINE_TESTOP(test_and_set_bits, or, LWSYNC_ON_SMP, ISYNC_ON_SMP)
-DEFINE_TESTOP(test_and_set_bits_lock, or, "", ISYNC_ON_SMP)
-DEFINE_TESTOP(test_and_clear_bits, andc, LWSYNC_ON_SMP, ISYNC_ON_SMP)
-DEFINE_TESTOP(test_and_change_bits, xor, LWSYNC_ON_SMP, ISYNC_ON_SMP)
+DEFINE_TESTOP(test_and_set_bits, or, LWSYNC_ON_SMP, ISYNC_ON_SMP, 0)
+DEFINE_TESTOP(test_and_set_bits_lock, or, "", ISYNC_ON_SMP, 1)
+DEFINE_TESTOP(test_and_clear_bits, andc, LWSYNC_ON_SMP, ISYNC_ON_SMP, 0)
+DEFINE_TESTOP(test_and_change_bits, xor, LWSYNC_ON_SMP, ISYNC_ON_SMP, 0)
 
 static __inline__ int test_and_set_bit(unsigned long nr,
                                       volatile unsigned long *addr)

diff --git a/arch/powerpc/include/asm/local.h b/arch/powerpc/include/asm/local.h
index 84b457a3c1bcda476a0c475d2bddf594a134e9bc..ce58c80e1bcf0f4cb153c18de99135686e4f3d04 100644 (file)
--- a/arch/powerpc/include/asm/local.h
+++ b/arch/powerpc/include/asm/local.h
@@ -24,7 +24,7 @@ static __inline__ long local_add_return(long a, local_t *l)
        long t;
 
        __asm__ __volatile__(
-"1:"   PPC_LLARX       "%0,0,%2                # local_add_return\n\
+"1:"   PPC_LLARX(%0,0,%2,0) "                  # local_add_return\n\
        add     %0,%1,%0\n"
        PPC405_ERR77(0,%2)
        PPC_STLCX       "%0,0,%2 \n\
@@ -43,7 +43,7 @@ static __inline__ long local_sub_return(long a, local_t *l)
        long t;
 
        __asm__ __volatile__(
-"1:"   PPC_LLARX       "%0,0,%2                # local_sub_return\n\
+"1:"   PPC_LLARX(%0,0,%2,0) "                  # local_sub_return\n\
        subf    %0,%1,%0\n"
        PPC405_ERR77(0,%2)
        PPC_STLCX       "%0,0,%2 \n\
@@ -60,7 +60,7 @@ static __inline__ long local_inc_return(local_t *l)
        long t;
 
        __asm__ __volatile__(
-"1:"   PPC_LLARX       "%0,0,%1                # local_inc_return\n\
+"1:"   PPC_LLARX(%0,0,%1,0) "                  # local_inc_return\n\
        addic   %0,%0,1\n"
        PPC405_ERR77(0,%1)
        PPC_STLCX       "%0,0,%1 \n\
@@ -87,7 +87,7 @@ static __inline__ long local_dec_return(local_t *l)
        long t;
 
        __asm__ __volatile__(
-"1:"   PPC_LLARX       "%0,0,%1                # local_dec_return\n\
+"1:"   PPC_LLARX(%0,0,%1,0) "                  # local_dec_return\n\
        addic   %0,%0,-1\n"
        PPC405_ERR77(0,%1)
        PPC_STLCX       "%0,0,%1\n\
@@ -117,7 +117,7 @@ static __inline__ int local_add_unless(local_t *l, long a, long u)
        long t;
 
        __asm__ __volatile__ (
-"1:"   PPC_LLARX       "%0,0,%1                # local_add_unless\n\
+"1:"   PPC_LLARX(%0,0,%1,0) "                  # local_add_unless\n\
        cmpw    0,%0,%3 \n\
        beq-    2f \n\
        add     %0,%2,%0 \n"
@@ -147,7 +147,7 @@ static __inline__ long local_dec_if_positive(local_t *l)
        long t;
 
        __asm__ __volatile__(
-"1:"   PPC_LLARX       "%0,0,%1                # local_dec_if_positive\n\
+"1:"   PPC_LLARX(%0,0,%1,0) "                  # local_dec_if_positive\n\
        cmpwi   %0,1\n\
        addi    %0,%0,-1\n\
        blt-    2f\n"

diff --git a/arch/powerpc/include/asm/ppc-opcode.h b/arch/powerpc/include/asm/ppc-opcode.h
index ecec76051184a7d36b7214c9b1c393cf604277f1..aea71479759030651b24361b7eb965fefa65772c 100644 (file)
--- a/arch/powerpc/include/asm/ppc-opcode.h
+++ b/arch/powerpc/include/asm/ppc-opcode.h
@@ -22,6 +22,7 @@
 #define PPC_INST_DCBZL                 0x7c2007ec
 #define PPC_INST_ISEL                  0x7c00001e
 #define PPC_INST_ISEL_MASK             0xfc00003e
+#define PPC_INST_LDARX                 0x7c0000a8
 #define PPC_INST_LSWI                  0x7c0004aa
 #define PPC_INST_LSWX                  0x7c00042a
 #define PPC_INST_LWARX                 0x7c000029
@@ -75,6 +76,9 @@
                                        __PPC_RA(a) | __PPC_RB(b))
 #define        PPC_DCBZL(a, b)         stringify_in_c(.long PPC_INST_DCBZL | \
                                        __PPC_RA(a) | __PPC_RB(b))
+#define PPC_LDARX(t, a, b, eh) stringify_in_c(.long PPC_INST_LDARX | \
+                                       __PPC_RT(t) | __PPC_RA(a) | \
+                                       __PPC_RB(b) | __PPC_EH(eh))
 #define PPC_LWARX(t, a, b, eh) stringify_in_c(.long PPC_INST_LWARX | \
                                        __PPC_RT(t) | __PPC_RA(a) | \
                                        __PPC_RB(b) | __PPC_EH(eh))