ipv6: Refactor update of IPv6 flowi destination address for srcrt (RH) option

[net-next-2.6.git] / net / core / dev.c

diff --git a/net/core/dev.c b/net/core/dev.c
index d273e4e3ecdc8e1fc4ddd4b8eb6695d4603e410e..ffca5c1066fa50947c4e5b88a86a47cace0ec57b 100644 (file)
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -954,18 +954,22 @@ int dev_alloc_name(struct net_device *dev, const char *name)
 }
 EXPORT_SYMBOL(dev_alloc_name);
 
-static int dev_get_valid_name(struct net *net, const char *name, char *buf,
-                             bool fmt)
+static int dev_get_valid_name(struct net_device *dev, const char *name, bool fmt)
 {
+       struct net *net;
+
+       BUG_ON(!dev_net(dev));
+       net = dev_net(dev);
+
        if (!dev_valid_name(name))
                return -EINVAL;
 
        if (fmt && strchr(name, '%'))
-               return __dev_alloc_name(net, name, buf);
+               return dev_alloc_name(dev, name);
        else if (__dev_get_by_name(net, name))
                return -EEXIST;
-       else if (buf != name)
-               strlcpy(buf, name, IFNAMSIZ);
+       else if (dev->name != name)
+               strlcpy(dev->name, name, IFNAMSIZ);
 
        return 0;
 }
@@ -997,7 +1001,7 @@ int dev_change_name(struct net_device *dev, const char *newname)
 
        memcpy(oldname, dev->name, IFNAMSIZ);
 
-       err = dev_get_valid_name(net, newname, dev->name, 1);
+       err = dev_get_valid_name(dev, newname, 1);
        if (err < 0)
                return err;
 
@@ -1573,7 +1577,9 @@ EXPORT_SYMBOL(__netif_schedule);
 
 void dev_kfree_skb_irq(struct sk_buff *skb)
 {
-       if (atomic_dec_and_test(&skb->users)) {
+       if (!skb->destructor)
+               dev_kfree_skb(skb);
+       else if (atomic_dec_and_test(&skb->users)) {
                struct softnet_data *sd;
                unsigned long flags;
 
@@ -2034,14 +2040,24 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
                                 struct netdev_queue *txq)
 {
        spinlock_t *root_lock = qdisc_lock(q);
+       bool contended = qdisc_is_running(q);
        int rc;
 
+       /*
+        * Heuristic to force contended enqueues to serialize on a
+        * separate lock before trying to get qdisc main lock.
+        * This permits __QDISC_STATE_RUNNING owner to get the lock more often
+        * and dequeue packets faster.
+        */
+       if (unlikely(contended))
+               spin_lock(&q->busylock);
+
        spin_lock(root_lock);
        if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
                kfree_skb(skb);
                rc = NET_XMIT_DROP;
        } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
-                  !test_and_set_bit(__QDISC_STATE_RUNNING, &q->state)) {
+                  qdisc_run_begin(q)) {
                /*
                 * This is a work-conserving queue; there are no old skbs
                 * waiting to be sent out; and the qdisc is not running -
@@ -2050,19 +2066,30 @@ static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
                if (!(dev->priv_flags & IFF_XMIT_DST_RELEASE))
                        skb_dst_force(skb);
                __qdisc_update_bstats(q, skb->len);
-               if (sch_direct_xmit(skb, q, dev, txq, root_lock))
+               if (sch_direct_xmit(skb, q, dev, txq, root_lock)) {
+                       if (unlikely(contended)) {
+                               spin_unlock(&q->busylock);
+                               contended = false;
+                       }
                        __qdisc_run(q);
-               else
-                       clear_bit(__QDISC_STATE_RUNNING, &q->state);
+               } else
+                       qdisc_run_end(q);
 
                rc = NET_XMIT_SUCCESS;
        } else {
                skb_dst_force(skb);
                rc = qdisc_enqueue_root(skb, q);
-               qdisc_run(q);
+               if (qdisc_run_begin(q)) {
+                       if (unlikely(contended)) {
+                               spin_unlock(&q->busylock);
+                               contended = false;
+                       }
+                       __qdisc_run(q);
+               }
        }
        spin_unlock(root_lock);
-
+       if (unlikely(contended))
+               spin_unlock(&q->busylock);
        return rc;
 }
 
@@ -2421,10 +2448,7 @@ static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
                if (skb_queue_len(&sd->input_pkt_queue)) {
 enqueue:
                        __skb_queue_tail(&sd->input_pkt_queue, skb);
-#ifdef CONFIG_RPS
-                       *qtail = sd->input_queue_head +
-                                       skb_queue_len(&sd->input_pkt_queue);
-#endif
+                       input_queue_tail_incr_save(sd, qtail);
                        rps_unlock(sd);
                        local_irq_restore(flags);
                        return NET_RX_SUCCESS;
@@ -2821,8 +2845,7 @@ static int __netif_receive_skb(struct sk_buff *skb)
                        skb->dev = master;
        }
 
-       __get_cpu_var(softnet_data).processed++;
-
+       __this_cpu_inc(softnet_data.processed);
        skb_reset_network_header(skb);
        skb_reset_transport_header(skb);
        skb->mac_len = skb->network_header - skb->mac_header;
@@ -2959,7 +2982,7 @@ static void flush_backlog(void *arg)
                if (skb->dev == dev) {
                        __skb_unlink(skb, &sd->input_pkt_queue);
                        kfree_skb(skb);
-                       input_queue_head_add(sd, 1);
+                       input_queue_head_incr(sd);
                }
        }
        rps_unlock(sd);
@@ -2968,6 +2991,7 @@ static void flush_backlog(void *arg)
                if (skb->dev == dev) {
                        __skb_unlink(skb, &sd->process_queue);
                        kfree_skb(skb);
+                       input_queue_head_incr(sd);
                }
        }
 }
@@ -3323,18 +3347,20 @@ static int process_backlog(struct napi_struct *napi, int quota)
                while ((skb = __skb_dequeue(&sd->process_queue))) {
                        local_irq_enable();
                        __netif_receive_skb(skb);
-                       if (++work >= quota)
-                               return work;
                        local_irq_disable();
+                       input_queue_head_incr(sd);
+                       if (++work >= quota) {
+                               local_irq_enable();
+                               return work;
+                       }
                }
 
                rps_lock(sd);
                qlen = skb_queue_len(&sd->input_pkt_queue);
-               if (qlen) {
-                       input_queue_head_add(sd, qlen);
+               if (qlen)
                        skb_queue_splice_tail_init(&sd->input_pkt_queue,
                                                   &sd->process_queue);
-               }
+
                if (qlen < quota - work) {
                        /*
                         * Inline a custom version of __napi_complete().
@@ -4960,7 +4986,7 @@ int register_netdevice(struct net_device *dev)
                }
        }
 
-       ret = dev_get_valid_name(net, dev->name, dev->name, 0);
+       ret = dev_get_valid_name(dev, dev->name, 0);
        if (ret)
                goto err_uninit;
 
@@ -5558,7 +5584,7 @@ int dev_change_net_namespace(struct net_device *dev, struct net *net, const char
                /* We get here if we can't use the current device name */
                if (!pat)
                        goto out;
-               if (dev_get_valid_name(net, pat, dev->name, 1))
+               if (dev_get_valid_name(dev, pat, 1))
                        goto out;
        }
 
@@ -5661,12 +5687,14 @@ static int dev_cpu_callback(struct notifier_block *nfb,
        local_irq_enable();
 
        /* Process offline CPU's input_pkt_queue */
-       while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {
+       while ((skb = __skb_dequeue(&oldsd->process_queue))) {
                netif_rx(skb);
-               input_queue_head_add(oldsd, 1);
+               input_queue_head_incr(oldsd);
        }
-       while ((skb = __skb_dequeue(&oldsd->process_queue)))
+       while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {
                netif_rx(skb);
+               input_queue_head_incr(oldsd);
+       }
 
        return NOTIFY_OK;
 }