udp: use atomic_inc_not_zero_hint

UDP sockets refcount is usually 2, unless an incoming frame is going to
be queued in receive or backlog queue.

Using atomic_inc_not_zero_hint() permits to reduce latency, because
processor issues less memory transactions.

Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/net/sock.h b/include/net/sock.h
index a6338d039857bf6b00bcff087f974ebf46ae0875..eb0c1f5046784e8819ec8e4fccc09c7f02f92288 100644 (file)
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -57,7 +57,7 @@
 #include <linux/rculist_nulls.h>
 #include <linux/poll.h>
 
-#include <asm/atomic.h>
+#include <linux/atomic.h>
 #include <net/dst.h>
 #include <net/checksum.h>
 

diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index 5e0a3a582a59a05466468b244371dc5675bedc67..491ecd3f7a01faf749a8f30e4da0b013970fe897 100644 (file)
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -430,7 +430,7 @@ begin:
 
        if (result) {
 exact_match:
-               if (unlikely(!atomic_inc_not_zero(&result->sk_refcnt)))
+               if (unlikely(!atomic_inc_not_zero_hint(&result->sk_refcnt, 2)))
                        result = NULL;
                else if (unlikely(compute_score2(result, net, saddr, sport,
                                  daddr, hnum, dif) < badness)) {
@@ -500,7 +500,7 @@ begin:
                goto begin;
 
        if (result) {
-               if (unlikely(!atomic_inc_not_zero(&result->sk_refcnt)))
+               if (unlikely(!atomic_inc_not_zero_hint(&result->sk_refcnt, 2)))
                        result = NULL;
                else if (unlikely(compute_score(result, net, saddr, hnum, sport,
                                  daddr, dport, dif) < badness)) {

diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c
index 91def93bec85060e7571218c20439cabd4ec824f..b541a4e009fba179d939cfe71b4d083168831728 100644 (file)
--- a/net/ipv6/udp.c
+++ b/net/ipv6/udp.c
@@ -227,7 +227,7 @@ begin:
 
        if (result) {
 exact_match:
-               if (unlikely(!atomic_inc_not_zero(&result->sk_refcnt)))
+               if (unlikely(!atomic_inc_not_zero_hint(&result->sk_refcnt, 2)))
                        result = NULL;
                else if (unlikely(compute_score2(result, net, saddr, sport,
                                  daddr, hnum, dif) < badness)) {
@@ -294,7 +294,7 @@ begin:
                goto begin;
 
        if (result) {
-               if (unlikely(!atomic_inc_not_zero(&result->sk_refcnt)))
+               if (unlikely(!atomic_inc_not_zero_hint(&result->sk_refcnt, 2)))
                        result = NULL;
                else if (unlikely(compute_score(result, net, hnum, saddr, sport,
                                        daddr, dport, dif) < badness)) {