net/ipv4/route.c

   1 /*
   2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   3  *              operating system.  INET is implemented using the  BSD Socket
   4  *              interface as the means of communication with the user level.
   5  *
   6  *              ROUTE - implementation of the IP router.
   7  *
   8  * Authors:     Ross Biro
   9  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  10  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
  11  *              Linus Torvalds, <Linus.Torvalds@helsinki.fi>
  12  *              Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
  13  *
  14  * Fixes:
  15  *              Alan Cox        :       Verify area fixes.
  16  *              Alan Cox        :       cli() protects routing changes
  17  *              Rui Oliveira    :       ICMP routing table updates
  18  *              (rco@di.uminho.pt)      Routing table insertion and update
  19  *              Linus Torvalds  :       Rewrote bits to be sensible
  20  *              Alan Cox        :       Added BSD route gw semantics
  21  *              Alan Cox        :       Super /proc >4K
  22  *              Alan Cox        :       MTU in route table
  23  *              Alan Cox        :       MSS actually. Also added the window
  24  *                                      clamper.
  25  *              Sam Lantinga    :       Fixed route matching in rt_del()
  26  *              Alan Cox        :       Routing cache support.
  27  *              Alan Cox        :       Removed compatibility cruft.
  28  *              Alan Cox        :       RTF_REJECT support.
  29  *              Alan Cox        :       TCP irtt support.
  30  *              Jonathan Naylor :       Added Metric support.
  31  *      Miquel van Smoorenburg  :       BSD API fixes.
  32  *      Miquel van Smoorenburg  :       Metrics.
  33  *              Alan Cox        :       Use __u32 properly
  34  *              Alan Cox        :       Aligned routing errors more closely with BSD
  35  *                                      our system is still very different.
  36  *              Alan Cox        :       Faster /proc handling
  37  *      Alexey Kuznetsov        :       Massive rework to support tree based routing,
  38  *                                      routing caches and better behaviour.
  39  *
  40  *              Olaf Erb        :       irtt wasn't being copied right.
  41  *              Bjorn Ekwall    :       Kerneld route support.
  42  *              Alan Cox        :       Multicast fixed (I hope)
  43  *              Pavel Krauz     :       Limited broadcast fixed
  44  *              Mike McLagan    :       Routing by source
  45  *      Alexey Kuznetsov        :       End of old history. Split to fib.c and
  46  *                                      route.c and rewritten from scratch.
  47  *              Andi Kleen      :       Load-limit warning messages.
  48  *      Vitaly E. Lavrov        :       Transparent proxy revived after year coma.
  49  *      Vitaly E. Lavrov        :       Race condition in ip_route_input_slow.
  50  *      Tobias Ringstrom        :       Uninitialized res.type in ip_route_output_slow.
  51  *      Vladimir V. Ivanov      :       IP rule info (flowid) is really useful.
  52  *              Marc Boucher    :       routing by fwmark
  53  *      Robert Olsson           :       Added rt_cache statistics
  54  *      Arnaldo C. Melo         :       Convert proc stuff to seq_file
  55  *      Eric Dumazet            :       hashed spinlocks and rt_check_expire() fixes.
  56  *      Ilia Sotnikov           :       Ignore TOS on PMTUD and Redirect
  57  *      Ilia Sotnikov           :       Removed TOS from hash calculations
  58  *
  59  *              This program is free software; you can redistribute it and/or
  60  *              modify it under the terms of the GNU General Public License
  61  *              as published by the Free Software Foundation; either version
  62  *              2 of the License, or (at your option) any later version.
  63  */
  64
  65 #include <linux/module.h>
  66 #include <asm/uaccess.h>
  67 #include <asm/system.h>
  68 #include <linux/bitops.h>
  69 #include <linux/types.h>
  70 #include <linux/kernel.h>
  71 #include <linux/mm.h>
  72 #include <linux/bootmem.h>
  73 #include <linux/string.h>
  74 #include <linux/socket.h>
  75 #include <linux/sockios.h>
  76 #include <linux/errno.h>
  77 #include <linux/in.h>
  78 #include <linux/inet.h>
  79 #include <linux/netdevice.h>
  80 #include <linux/proc_fs.h>
  81 #include <linux/init.h>
  82 #include <linux/workqueue.h>
  83 #include <linux/skbuff.h>
  84 #include <linux/inetdevice.h>
  85 #include <linux/igmp.h>
  86 #include <linux/pkt_sched.h>
  87 #include <linux/mroute.h>
  88 #include <linux/netfilter_ipv4.h>
  89 #include <linux/random.h>
  90 #include <linux/jhash.h>
  91 #include <linux/rcupdate.h>
  92 #include <linux/times.h>
  93 #include <linux/slab.h>
  94 #include <net/dst.h>
  95 #include <net/net_namespace.h>
  96 #include <net/protocol.h>
  97 #include <net/ip.h>
  98 #include <net/route.h>
  99 #include <net/inetpeer.h>
 100 #include <net/sock.h>
 101 #include <net/ip_fib.h>
 102 #include <net/arp.h>
 103 #include <net/tcp.h>
 104 #include <net/icmp.h>
 105 #include <net/xfrm.h>
 106 #include <net/netevent.h>
 107 #include <net/rtnetlink.h>
 108 #ifdef CONFIG_SYSCTL
 109 #include <linux/sysctl.h>
 110 #endif
 111
 112 #define RT_FL_TOS(oldflp) \
 113     ((u32)(oldflp->fl4_tos & (IPTOS_RT_MASK | RTO_ONLINK)))
 114
 115 #define IP_MAX_MTU      0xFFF0
 116
 117 #define RT_GC_TIMEOUT (300*HZ)
 118
 119 static int ip_rt_max_size;
 120 static int ip_rt_gc_timeout __read_mostly       = RT_GC_TIMEOUT;
 121 static int ip_rt_gc_interval __read_mostly      = 60 * HZ;
 122 static int ip_rt_gc_min_interval __read_mostly  = HZ / 2;
 123 static int ip_rt_redirect_number __read_mostly  = 9;
 124 static int ip_rt_redirect_load __read_mostly    = HZ / 50;
 125 static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1));
 126 static int ip_rt_error_cost __read_mostly       = HZ;
 127 static int ip_rt_error_burst __read_mostly      = 5 * HZ;
 128 static int ip_rt_gc_elasticity __read_mostly    = 8;
 129 static int ip_rt_mtu_expires __read_mostly      = 10 * 60 * HZ;
 130 static int ip_rt_min_pmtu __read_mostly         = 512 + 20 + 20;
 131 static int ip_rt_min_advmss __read_mostly       = 256;
 132 static int rt_chain_length_max __read_mostly    = 20;
 133
 134 static struct delayed_work expires_work;
 135 static unsigned long expires_ljiffies;
 136
 137 /*
 138  *      Interface to generic destination cache.
 139  */
 140
 141 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
 142 static void              ipv4_dst_destroy(struct dst_entry *dst);
 143 static void              ipv4_dst_ifdown(struct dst_entry *dst,
 144                                          struct net_device *dev, int how);
 145 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
 146 static void              ipv4_link_failure(struct sk_buff *skb);
 147 static void              ip_rt_update_pmtu(struct dst_entry *dst, u32 mtu);
 148 static int rt_garbage_collect(struct dst_ops *ops);
 149
 150
 151 static struct dst_ops ipv4_dst_ops = {
 152         .family =               AF_INET,
 153         .protocol =             cpu_to_be16(ETH_P_IP),
 154         .gc =                   rt_garbage_collect,
 155         .check =                ipv4_dst_check,
 156         .destroy =              ipv4_dst_destroy,
 157         .ifdown =               ipv4_dst_ifdown,
 158         .negative_advice =      ipv4_negative_advice,
 159         .link_failure =         ipv4_link_failure,
 160         .update_pmtu =          ip_rt_update_pmtu,
 161         .local_out =            __ip_local_out,
 162         .entries =              ATOMIC_INIT(0),
 163 };
 164
 165 #define ECN_OR_COST(class)      TC_PRIO_##class
 166
 167 const __u8 ip_tos2prio[16] = {
 168         TC_PRIO_BESTEFFORT,
 169         ECN_OR_COST(FILLER),
 170         TC_PRIO_BESTEFFORT,
 171         ECN_OR_COST(BESTEFFORT),
 172         TC_PRIO_BULK,
 173         ECN_OR_COST(BULK),
 174         TC_PRIO_BULK,
 175         ECN_OR_COST(BULK),
 176         TC_PRIO_INTERACTIVE,
 177         ECN_OR_COST(INTERACTIVE),
 178         TC_PRIO_INTERACTIVE,
 179         ECN_OR_COST(INTERACTIVE),
 180         TC_PRIO_INTERACTIVE_BULK,
 181         ECN_OR_COST(INTERACTIVE_BULK),
 182         TC_PRIO_INTERACTIVE_BULK,
 183         ECN_OR_COST(INTERACTIVE_BULK)
 184 };
 185
 186
 187 /*
 188  * Route cache.
 189  */
 190
 191 /* The locking scheme is rather straight forward:
 192  *
 193  * 1) Read-Copy Update protects the buckets of the central route hash.
 194  * 2) Only writers remove entries, and they hold the lock
 195  *    as they look at rtable reference counts.
 196  * 3) Only readers acquire references to rtable entries,
 197  *    they do so with atomic increments and with the
 198  *    lock held.
 199  */
 200
 201 struct rt_hash_bucket {
 202         struct rtable   *chain;
 203 };
 204
 205 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) || \
 206         defined(CONFIG_PROVE_LOCKING)
 207 /*
 208  * Instead of using one spinlock for each rt_hash_bucket, we use a table of spinlocks
 209  * The size of this table is a power of two and depends on the number of CPUS.
 210  * (on lockdep we have a quite big spinlock_t, so keep the size down there)
 211  */
 212 #ifdef CONFIG_LOCKDEP
 213 # define RT_HASH_LOCK_SZ        256
 214 #else
 215 # if NR_CPUS >= 32
 216 #  define RT_HASH_LOCK_SZ       4096
 217 # elif NR_CPUS >= 16
 218 #  define RT_HASH_LOCK_SZ       2048
 219 # elif NR_CPUS >= 8
 220 #  define RT_HASH_LOCK_SZ       1024
 221 # elif NR_CPUS >= 4
 222 #  define RT_HASH_LOCK_SZ       512
 223 # else
 224 #  define RT_HASH_LOCK_SZ       256
 225 # endif
 226 #endif
 227
 228 static spinlock_t       *rt_hash_locks;
 229 # define rt_hash_lock_addr(slot) &rt_hash_locks[(slot) & (RT_HASH_LOCK_SZ - 1)]
 230
 231 static __init void rt_hash_lock_init(void)
 232 {
 233         int i;
 234
 235         rt_hash_locks = kmalloc(sizeof(spinlock_t) * RT_HASH_LOCK_SZ,
 236                         GFP_KERNEL);
 237         if (!rt_hash_locks)
 238                 panic("IP: failed to allocate rt_hash_locks\n");
 239
 240         for (i = 0; i < RT_HASH_LOCK_SZ; i++)
 241                 spin_lock_init(&rt_hash_locks[i]);
 242 }
 243 #else
 244 # define rt_hash_lock_addr(slot) NULL
 245
 246 static inline void rt_hash_lock_init(void)
 247 {
 248 }
 249 #endif
 250
 251 static struct rt_hash_bucket    *rt_hash_table __read_mostly;
 252 static unsigned                 rt_hash_mask __read_mostly;
 253 static unsigned int             rt_hash_log  __read_mostly;
 254
 255 static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
 256 #define RT_CACHE_STAT_INC(field) __this_cpu_inc(rt_cache_stat.field)
 257
 258 static inline unsigned int rt_hash(__be32 daddr, __be32 saddr, int idx,
 259                                    int genid)
 260 {
 261         return jhash_3words((__force u32)daddr, (__force u32)saddr,
 262                             idx, genid)
 263                 & rt_hash_mask;
 264 }
 265
 266 static inline int rt_genid(struct net *net)
 267 {
 268         return atomic_read(&net->ipv4.rt_genid);
 269 }
 270
 271 #ifdef CONFIG_PROC_FS
 272 struct rt_cache_iter_state {
 273         struct seq_net_private p;
 274         int bucket;
 275         int genid;
 276 };
 277
 278 static struct rtable *rt_cache_get_first(struct seq_file *seq)
 279 {
 280         struct rt_cache_iter_state *st = seq->private;
 281         struct rtable *r = NULL;
 282
 283         for (st->bucket = rt_hash_mask; st->bucket >= 0; --st->bucket) {
 284                 if (!rt_hash_table[st->bucket].chain)
 285                         continue;
 286                 rcu_read_lock_bh();
 287                 r = rcu_dereference_bh(rt_hash_table[st->bucket].chain);
 288                 while (r) {
 289                         if (dev_net(r->dst.dev) == seq_file_net(seq) &&
 290                             r->rt_genid == st->genid)
 291                                 return r;
 292                         r = rcu_dereference_bh(r->dst.rt_next);
 293                 }
 294                 rcu_read_unlock_bh();
 295         }
 296         return r;
 297 }
 298
 299 static struct rtable *__rt_cache_get_next(struct seq_file *seq,
 300                                           struct rtable *r)
 301 {
 302         struct rt_cache_iter_state *st = seq->private;
 303
 304         r = r->dst.rt_next;
 305         while (!r) {
 306                 rcu_read_unlock_bh();
 307                 do {
 308                         if (--st->bucket < 0)
 309                                 return NULL;
 310                 } while (!rt_hash_table[st->bucket].chain);
 311                 rcu_read_lock_bh();
 312                 r = rt_hash_table[st->bucket].chain;
 313         }
 314         return rcu_dereference_bh(r);
 315 }
 316
 317 static struct rtable *rt_cache_get_next(struct seq_file *seq,
 318                                         struct rtable *r)
 319 {
 320         struct rt_cache_iter_state *st = seq->private;
 321         while ((r = __rt_cache_get_next(seq, r)) != NULL) {
 322                 if (dev_net(r->dst.dev) != seq_file_net(seq))
 323                         continue;
 324                 if (r->rt_genid == st->genid)
 325                         break;
 326         }
 327         return r;
 328 }
 329
 330 static struct rtable *rt_cache_get_idx(struct seq_file *seq, loff_t pos)
 331 {
 332         struct rtable *r = rt_cache_get_first(seq);
 333
 334         if (r)
 335                 while (pos && (r = rt_cache_get_next(seq, r)))
 336                         --pos;
 337         return pos ? NULL : r;
 338 }
 339
 340 static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
 341 {
 342         struct rt_cache_iter_state *st = seq->private;
 343         if (*pos)
 344                 return rt_cache_get_idx(seq, *pos - 1);
 345         st->genid = rt_genid(seq_file_net(seq));
 346         return SEQ_START_TOKEN;
 347 }
 348
 349 static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 350 {
 351         struct rtable *r;
 352
 353         if (v == SEQ_START_TOKEN)
 354                 r = rt_cache_get_first(seq);
 355         else
 356                 r = rt_cache_get_next(seq, v);
 357         ++*pos;
 358         return r;
 359 }
 360
 361 static void rt_cache_seq_stop(struct seq_file *seq, void *v)
 362 {
 363         if (v && v != SEQ_START_TOKEN)
 364                 rcu_read_unlock_bh();
 365 }
 366
 367 static int rt_cache_seq_show(struct seq_file *seq, void *v)
 368 {
 369         if (v == SEQ_START_TOKEN)
 370                 seq_printf(seq, "%-127s\n",
 371                            "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
 372                            "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
 373                            "HHUptod\tSpecDst");
 374         else {
 375                 struct rtable *r = v;
 376                 int len;
 377
 378                 seq_printf(seq, "%s\t%08X\t%08X\t%8X\t%d\t%u\t%d\t"
 379                               "%08X\t%d\t%u\t%u\t%02X\t%d\t%1d\t%08X%n",
 380                         r->dst.dev ? r->dst.dev->name : "*",
 381                         (__force u32)r->rt_dst,
 382                         (__force u32)r->rt_gateway,
 383                         r->rt_flags, atomic_read(&r->dst.__refcnt),
 384                         r->dst.__use, 0, (__force u32)r->rt_src,
 385                         (dst_metric(&r->dst, RTAX_ADVMSS) ?
 386                              (int)dst_metric(&r->dst, RTAX_ADVMSS) + 40 : 0),
 387                         dst_metric(&r->dst, RTAX_WINDOW),
 388                         (int)((dst_metric(&r->dst, RTAX_RTT) >> 3) +
 389                               dst_metric(&r->dst, RTAX_RTTVAR)),
 390                         r->fl.fl4_tos,
 391                         r->dst.hh ? atomic_read(&r->dst.hh->hh_refcnt) : -1,
 392                         r->dst.hh ? (r->dst.hh->hh_output ==
 393                                        dev_queue_xmit) : 0,
 394                         r->rt_spec_dst, &len);
 395
 396                 seq_printf(seq, "%*s\n", 127 - len, "");
 397         }
 398         return 0;
 399 }
 400
 401 static const struct seq_operations rt_cache_seq_ops = {
 402         .start  = rt_cache_seq_start,
 403         .next   = rt_cache_seq_next,
 404         .stop   = rt_cache_seq_stop,
 405         .show   = rt_cache_seq_show,
 406 };
 407
 408 static int rt_cache_seq_open(struct inode *inode, struct file *file)
 409 {
 410         return seq_open_net(inode, file, &rt_cache_seq_ops,
 411                         sizeof(struct rt_cache_iter_state));
 412 }
 413
 414 static const struct file_operations rt_cache_seq_fops = {
 415         .owner   = THIS_MODULE,
 416         .open    = rt_cache_seq_open,
 417         .read    = seq_read,
 418         .llseek  = seq_lseek,
 419         .release = seq_release_net,
 420 };
 421
 422
 423 static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
 424 {
 425         int cpu;
 426
 427         if (*pos == 0)
 428                 return SEQ_START_TOKEN;
 429
 430         for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
 431                 if (!cpu_possible(cpu))
 432                         continue;
 433                 *pos = cpu+1;
 434                 return &per_cpu(rt_cache_stat, cpu);
 435         }
 436         return NULL;
 437 }
 438
 439 static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 440 {
 441         int cpu;
 442
 443         for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
 444                 if (!cpu_possible(cpu))
 445                         continue;
 446                 *pos = cpu+1;
 447                 return &per_cpu(rt_cache_stat, cpu);
 448         }
 449         return NULL;
 450
 451 }
 452
 453 static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
 454 {
 455
 456 }
 457
 458 static int rt_cpu_seq_show(struct seq_file *seq, void *v)
 459 {
 460         struct rt_cache_stat *st = v;
 461
 462         if (v == SEQ_START_TOKEN) {
 463                 seq_printf(seq, "entries  in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src  out_hit out_slow_tot out_slow_mc  gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
 464                 return 0;
 465         }
 466
 467         seq_printf(seq,"%08x  %08x %08x %08x %08x %08x %08x %08x "
 468                    " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
 469                    atomic_read(&ipv4_dst_ops.entries),
 470                    st->in_hit,
 471                    st->in_slow_tot,
 472                    st->in_slow_mc,
 473                    st->in_no_route,
 474                    st->in_brd,
 475                    st->in_martian_dst,
 476                    st->in_martian_src,
 477
 478                    st->out_hit,
 479                    st->out_slow_tot,
 480                    st->out_slow_mc,
 481
 482                    st->gc_total,
 483                    st->gc_ignored,
 484                    st->gc_goal_miss,
 485                    st->gc_dst_overflow,
 486                    st->in_hlist_search,
 487                    st->out_hlist_search
 488                 );
 489         return 0;
 490 }
 491
 492 static const struct seq_operations rt_cpu_seq_ops = {
 493         .start  = rt_cpu_seq_start,
 494         .next   = rt_cpu_seq_next,
 495         .stop   = rt_cpu_seq_stop,
 496         .show   = rt_cpu_seq_show,
 497 };
 498
 499
 500 static int rt_cpu_seq_open(struct inode *inode, struct file *file)
 501 {
 502         return seq_open(file, &rt_cpu_seq_ops);
 503 }
 504
 505 static const struct file_operations rt_cpu_seq_fops = {
 506         .owner   = THIS_MODULE,
 507         .open    = rt_cpu_seq_open,
 508         .read    = seq_read,
 509         .llseek  = seq_lseek,
 510         .release = seq_release,
 511 };
 512
 513 #ifdef CONFIG_NET_CLS_ROUTE
 514 static int rt_acct_proc_show(struct seq_file *m, void *v)
 515 {
 516         struct ip_rt_acct *dst, *src;
 517         unsigned int i, j;
 518
 519         dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
 520         if (!dst)
 521                 return -ENOMEM;
 522
 523         for_each_possible_cpu(i) {
 524                 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
 525                 for (j = 0; j < 256; j++) {
 526                         dst[j].o_bytes   += src[j].o_bytes;
 527                         dst[j].o_packets += src[j].o_packets;
 528                         dst[j].i_bytes   += src[j].i_bytes;
 529                         dst[j].i_packets += src[j].i_packets;
 530                 }
 531         }
 532
 533         seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
 534         kfree(dst);
 535         return 0;
 536 }
 537
 538 static int rt_acct_proc_open(struct inode *inode, struct file *file)
 539 {
 540         return single_open(file, rt_acct_proc_show, NULL);
 541 }
 542
 543 static const struct file_operations rt_acct_proc_fops = {
 544         .owner          = THIS_MODULE,
 545         .open           = rt_acct_proc_open,
 546         .read           = seq_read,
 547         .llseek         = seq_lseek,
 548         .release        = single_release,
 549 };
 550 #endif
 551
 552 static int __net_init ip_rt_do_proc_init(struct net *net)
 553 {
 554         struct proc_dir_entry *pde;
 555
 556         pde = proc_net_fops_create(net, "rt_cache", S_IRUGO,
 557                         &rt_cache_seq_fops);
 558         if (!pde)
 559                 goto err1;
 560
 561         pde = proc_create("rt_cache", S_IRUGO,
 562                           net->proc_net_stat, &rt_cpu_seq_fops);
 563         if (!pde)
 564                 goto err2;
 565
 566 #ifdef CONFIG_NET_CLS_ROUTE
 567         pde = proc_create("rt_acct", 0, net->proc_net, &rt_acct_proc_fops);
 568         if (!pde)
 569                 goto err3;
 570 #endif
 571         return 0;
 572
 573 #ifdef CONFIG_NET_CLS_ROUTE
 574 err3:
 575         remove_proc_entry("rt_cache", net->proc_net_stat);
 576 #endif
 577 err2:
 578         remove_proc_entry("rt_cache", net->proc_net);
 579 err1:
 580         return -ENOMEM;
 581 }
 582
 583 static void __net_exit ip_rt_do_proc_exit(struct net *net)
 584 {
 585         remove_proc_entry("rt_cache", net->proc_net_stat);
 586         remove_proc_entry("rt_cache", net->proc_net);
 587 #ifdef CONFIG_NET_CLS_ROUTE
 588         remove_proc_entry("rt_acct", net->proc_net);
 589 #endif
 590 }
 591
 592 static struct pernet_operations ip_rt_proc_ops __net_initdata =  {
 593         .init = ip_rt_do_proc_init,
 594         .exit = ip_rt_do_proc_exit,
 595 };
 596
 597 static int __init ip_rt_proc_init(void)
 598 {
 599         return register_pernet_subsys(&ip_rt_proc_ops);
 600 }
 601
 602 #else
 603 static inline int ip_rt_proc_init(void)
 604 {
 605         return 0;
 606 }
 607 #endif /* CONFIG_PROC_FS */
 608
 609 static inline void rt_free(struct rtable *rt)
 610 {
 611         call_rcu_bh(&rt->dst.rcu_head, dst_rcu_free);
 612 }
 613
 614 static inline void rt_drop(struct rtable *rt)
 615 {
 616         ip_rt_put(rt);
 617         call_rcu_bh(&rt->dst.rcu_head, dst_rcu_free);
 618 }
 619
 620 static inline int rt_fast_clean(struct rtable *rth)
 621 {
 622         /* Kill broadcast/multicast entries very aggresively, if they
 623            collide in hash table with more useful entries */
 624         return (rth->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST)) &&
 625                 rth->fl.iif && rth->dst.rt_next;
 626 }
 627
 628 static inline int rt_valuable(struct rtable *rth)
 629 {
 630         return (rth->rt_flags & (RTCF_REDIRECTED | RTCF_NOTIFY)) ||
 631                 rth->dst.expires;
 632 }
 633
 634 static int rt_may_expire(struct rtable *rth, unsigned long tmo1, unsigned long tmo2)
 635 {
 636         unsigned long age;
 637         int ret = 0;
 638
 639         if (atomic_read(&rth->dst.__refcnt))
 640                 goto out;
 641
 642         ret = 1;
 643         if (rth->dst.expires &&
 644             time_after_eq(jiffies, rth->dst.expires))
 645                 goto out;
 646
 647         age = jiffies - rth->dst.lastuse;
 648         ret = 0;
 649         if ((age <= tmo1 && !rt_fast_clean(rth)) ||
 650             (age <= tmo2 && rt_valuable(rth)))
 651                 goto out;
 652         ret = 1;
 653 out:    return ret;
 654 }
 655
 656 /* Bits of score are:
 657  * 31: very valuable
 658  * 30: not quite useless
 659  * 29..0: usage counter
 660  */
 661 static inline u32 rt_score(struct rtable *rt)
 662 {
 663         u32 score = jiffies - rt->dst.lastuse;
 664
 665         score = ~score & ~(3<<30);
 666
 667         if (rt_valuable(rt))
 668                 score |= (1<<31);
 669
 670         if (!rt->fl.iif ||
 671             !(rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST|RTCF_LOCAL)))
 672                 score |= (1<<30);
 673
 674         return score;
 675 }
 676
 677 static inline bool rt_caching(const struct net *net)
 678 {
 679         return net->ipv4.current_rt_cache_rebuild_count <=
 680                 net->ipv4.sysctl_rt_cache_rebuild_count;
 681 }
 682
 683 static inline bool compare_hash_inputs(const struct flowi *fl1,
 684                                         const struct flowi *fl2)
 685 {
 686         return ((((__force u32)fl1->nl_u.ip4_u.daddr ^ (__force u32)fl2->nl_u.ip4_u.daddr) |
 687                 ((__force u32)fl1->nl_u.ip4_u.saddr ^ (__force u32)fl2->nl_u.ip4_u.saddr) |
 688                 (fl1->iif ^ fl2->iif)) == 0);
 689 }
 690
 691 static inline int compare_keys(struct flowi *fl1, struct flowi *fl2)
 692 {
 693         return (((__force u32)fl1->nl_u.ip4_u.daddr ^ (__force u32)fl2->nl_u.ip4_u.daddr) |
 694                 ((__force u32)fl1->nl_u.ip4_u.saddr ^ (__force u32)fl2->nl_u.ip4_u.saddr) |
 695                 (fl1->mark ^ fl2->mark) |
 696                 (*(u16 *)&fl1->nl_u.ip4_u.tos ^ *(u16 *)&fl2->nl_u.ip4_u.tos) |
 697                 (fl1->oif ^ fl2->oif) |
 698                 (fl1->iif ^ fl2->iif)) == 0;
 699 }
 700
 701 static inline int compare_netns(struct rtable *rt1, struct rtable *rt2)
 702 {
 703         return net_eq(dev_net(rt1->dst.dev), dev_net(rt2->dst.dev));
 704 }
 705
 706 static inline int rt_is_expired(struct rtable *rth)
 707 {
 708         return rth->rt_genid != rt_genid(dev_net(rth->dst.dev));
 709 }
 710
 711 /*
 712  * Perform a full scan of hash table and free all entries.
 713  * Can be called by a softirq or a process.
 714  * In the later case, we want to be reschedule if necessary
 715  */
 716 static void rt_do_flush(int process_context)
 717 {
 718         unsigned int i;
 719         struct rtable *rth, *next;
 720         struct rtable * tail;
 721
 722         for (i = 0; i <= rt_hash_mask; i++) {
 723                 if (process_context && need_resched())
 724                         cond_resched();
 725                 rth = rt_hash_table[i].chain;
 726                 if (!rth)
 727                         continue;
 728
 729                 spin_lock_bh(rt_hash_lock_addr(i));
 730 #ifdef CONFIG_NET_NS
 731                 {
 732                 struct rtable ** prev, * p;
 733
 734                 rth = rt_hash_table[i].chain;
 735
 736                 /* defer releasing the head of the list after spin_unlock */
 737                 for (tail = rth; tail; tail = tail->dst.rt_next)
 738                         if (!rt_is_expired(tail))
 739                                 break;
 740                 if (rth != tail)
 741                         rt_hash_table[i].chain = tail;
 742
 743                 /* call rt_free on entries after the tail requiring flush */
 744                 prev = &rt_hash_table[i].chain;
 745                 for (p = *prev; p; p = next) {
 746                         next = p->dst.rt_next;
 747                         if (!rt_is_expired(p)) {
 748                                 prev = &p->dst.rt_next;
 749                         } else {
 750                                 *prev = next;
 751                                 rt_free(p);
 752                         }
 753                 }
 754                 }
 755 #else
 756                 rth = rt_hash_table[i].chain;
 757                 rt_hash_table[i].chain = NULL;
 758                 tail = NULL;
 759 #endif
 760                 spin_unlock_bh(rt_hash_lock_addr(i));
 761
 762                 for (; rth != tail; rth = next) {
 763                         next = rth->dst.rt_next;
 764                         rt_free(rth);
 765                 }
 766         }
 767 }
 768
 769 /*
 770  * While freeing expired entries, we compute average chain length
 771  * and standard deviation, using fixed-point arithmetic.
 772  * This to have an estimation of rt_chain_length_max
 773  *  rt_chain_length_max = max(elasticity, AVG + 4*SD)
 774  * We use 3 bits for frational part, and 29 (or 61) for magnitude.
 775  */
 776
 777 #define FRACT_BITS 3
 778 #define ONE (1UL << FRACT_BITS)
 779
 780 /*
 781  * Given a hash chain and an item in this hash chain,
 782  * find if a previous entry has the same hash_inputs
 783  * (but differs on tos, mark or oif)
 784  * Returns 0 if an alias is found.
 785  * Returns ONE if rth has no alias before itself.
 786  */
 787 static int has_noalias(const struct rtable *head, const struct rtable *rth)
 788 {
 789         const struct rtable *aux = head;
 790
 791         while (aux != rth) {
 792                 if (compare_hash_inputs(&aux->fl, &rth->fl))
 793                         return 0;
 794                 aux = aux->dst.rt_next;
 795         }
 796         return ONE;
 797 }
 798
 799 static void rt_check_expire(void)
 800 {
 801         static unsigned int rover;
 802         unsigned int i = rover, goal;
 803         struct rtable *rth, **rthp;
 804         unsigned long samples = 0;
 805         unsigned long sum = 0, sum2 = 0;
 806         unsigned long delta;
 807         u64 mult;
 808
 809         delta = jiffies - expires_ljiffies;
 810         expires_ljiffies = jiffies;
 811         mult = ((u64)delta) << rt_hash_log;
 812         if (ip_rt_gc_timeout > 1)
 813                 do_div(mult, ip_rt_gc_timeout);
 814         goal = (unsigned int)mult;
 815         if (goal > rt_hash_mask)
 816                 goal = rt_hash_mask + 1;
 817         for (; goal > 0; goal--) {
 818                 unsigned long tmo = ip_rt_gc_timeout;
 819                 unsigned long length;
 820
 821                 i = (i + 1) & rt_hash_mask;
 822                 rthp = &rt_hash_table[i].chain;
 823
 824                 if (need_resched())
 825                         cond_resched();
 826
 827                 samples++;
 828
 829                 if (*rthp == NULL)
 830                         continue;
 831                 length = 0;
 832                 spin_lock_bh(rt_hash_lock_addr(i));
 833                 while ((rth = *rthp) != NULL) {
 834                         prefetch(rth->dst.rt_next);
 835                         if (rt_is_expired(rth)) {
 836                                 *rthp = rth->dst.rt_next;
 837                                 rt_free(rth);
 838                                 continue;
 839                         }
 840                         if (rth->dst.expires) {
 841                                 /* Entry is expired even if it is in use */
 842                                 if (time_before_eq(jiffies, rth->dst.expires)) {
 843 nofree:
 844                                         tmo >>= 1;
 845                                         rthp = &rth->dst.rt_next;
 846                                         /*
 847                                          * We only count entries on
 848                                          * a chain with equal hash inputs once
 849                                          * so that entries for different QOS
 850                                          * levels, and other non-hash input
 851                                          * attributes don't unfairly skew
 852                                          * the length computation
 853                                          */
 854                                         length += has_noalias(rt_hash_table[i].chain, rth);
 855                                         continue;
 856                                 }
 857                         } else if (!rt_may_expire(rth, tmo, ip_rt_gc_timeout))
 858                                 goto nofree;
 859
 860                         /* Cleanup aged off entries. */
 861                         *rthp = rth->dst.rt_next;
 862                         rt_free(rth);
 863                 }
 864                 spin_unlock_bh(rt_hash_lock_addr(i));
 865                 sum += length;
 866                 sum2 += length*length;
 867         }
 868         if (samples) {
 869                 unsigned long avg = sum / samples;
 870                 unsigned long sd = int_sqrt(sum2 / samples - avg*avg);
 871                 rt_chain_length_max = max_t(unsigned long,
 872                                         ip_rt_gc_elasticity,
 873                                         (avg + 4*sd) >> FRACT_BITS);
 874         }
 875         rover = i;
 876 }
 877
 878 /*
 879  * rt_worker_func() is run in process context.
 880  * we call rt_check_expire() to scan part of the hash table
 881  */
 882 static void rt_worker_func(struct work_struct *work)
 883 {
 884         rt_check_expire();
 885         schedule_delayed_work(&expires_work, ip_rt_gc_interval);
 886 }
 887
 888 /*
 889  * Pertubation of rt_genid by a small quantity [1..256]
 890  * Using 8 bits of shuffling ensure we can call rt_cache_invalidate()
 891  * many times (2^24) without giving recent rt_genid.
 892  * Jenkins hash is strong enough that litle changes of rt_genid are OK.
 893  */
 894 static void rt_cache_invalidate(struct net *net)
 895 {
 896         unsigned char shuffle;
 897
 898         get_random_bytes(&shuffle, sizeof(shuffle));
 899         atomic_add(shuffle + 1U, &net->ipv4.rt_genid);
 900 }
 901
 902 /*
 903  * delay < 0  : invalidate cache (fast : entries will be deleted later)
 904  * delay >= 0 : invalidate & flush cache (can be long)
 905  */
 906 void rt_cache_flush(struct net *net, int delay)
 907 {
 908         rt_cache_invalidate(net);
 909         if (delay >= 0)
 910                 rt_do_flush(!in_softirq());
 911 }
 912
 913 /* Flush previous cache invalidated entries from the cache */
 914 void rt_cache_flush_batch(void)
 915 {
 916         rt_do_flush(!in_softirq());
 917 }
 918
 919 static void rt_emergency_hash_rebuild(struct net *net)
 920 {
 921         if (net_ratelimit())
 922                 printk(KERN_WARNING "Route hash chain too long!\n");
 923         rt_cache_invalidate(net);
 924 }
 925
 926 /*
 927    Short description of GC goals.
 928
 929    We want to build algorithm, which will keep routing cache
 930    at some equilibrium point, when number of aged off entries
 931    is kept approximately equal to newly generated ones.
 932
 933    Current expiration strength is variable "expire".
 934    We try to adjust it dynamically, so that if networking
 935    is idle expires is large enough to keep enough of warm entries,
 936    and when load increases it reduces to limit cache size.
 937  */
 938
 939 static int rt_garbage_collect(struct dst_ops *ops)
 940 {
 941         static unsigned long expire = RT_GC_TIMEOUT;
 942         static unsigned long last_gc;
 943         static int rover;
 944         static int equilibrium;
 945         struct rtable *rth, **rthp;
 946         unsigned long now = jiffies;
 947         int goal;
 948
 949         /*
 950          * Garbage collection is pretty expensive,
 951          * do not make it too frequently.
 952          */
 953
 954         RT_CACHE_STAT_INC(gc_total);
 955
 956         if (now - last_gc < ip_rt_gc_min_interval &&
 957             atomic_read(&ipv4_dst_ops.entries) < ip_rt_max_size) {
 958                 RT_CACHE_STAT_INC(gc_ignored);
 959                 goto out;
 960         }
 961
 962         /* Calculate number of entries, which we want to expire now. */
 963         goal = atomic_read(&ipv4_dst_ops.entries) -
 964                 (ip_rt_gc_elasticity << rt_hash_log);
 965         if (goal <= 0) {
 966                 if (equilibrium < ipv4_dst_ops.gc_thresh)
 967                         equilibrium = ipv4_dst_ops.gc_thresh;
 968                 goal = atomic_read(&ipv4_dst_ops.entries) - equilibrium;
 969                 if (goal > 0) {
 970                         equilibrium += min_t(unsigned int, goal >> 1, rt_hash_mask + 1);
 971                         goal = atomic_read(&ipv4_dst_ops.entries) - equilibrium;
 972                 }
 973         } else {
 974                 /* We are in dangerous area. Try to reduce cache really
 975                  * aggressively.
 976                  */
 977                 goal = max_t(unsigned int, goal >> 1, rt_hash_mask + 1);
 978                 equilibrium = atomic_read(&ipv4_dst_ops.entries) - goal;
 979         }
 980
 981         if (now - last_gc >= ip_rt_gc_min_interval)
 982                 last_gc = now;
 983
 984         if (goal <= 0) {
 985                 equilibrium += goal;
 986                 goto work_done;
 987         }
 988
 989         do {
 990                 int i, k;
 991
 992                 for (i = rt_hash_mask, k = rover; i >= 0; i--) {
 993                         unsigned long tmo = expire;
 994
 995                         k = (k + 1) & rt_hash_mask;
 996                         rthp = &rt_hash_table[k].chain;
 997                         spin_lock_bh(rt_hash_lock_addr(k));
 998                         while ((rth = *rthp) != NULL) {
 999                                 if (!rt_is_expired(rth) &&
1000                                         !rt_may_expire(rth, tmo, expire)) {
1001                                         tmo >>= 1;
1002                                         rthp = &rth->dst.rt_next;
1003                                         continue;
1004                                 }
1005                                 *rthp = rth->dst.rt_next;
1006                                 rt_free(rth);
1007                                 goal--;
1008                         }
1009                         spin_unlock_bh(rt_hash_lock_addr(k));
1010                         if (goal <= 0)
1011                                 break;
1012                 }
1013                 rover = k;
1014
1015                 if (goal <= 0)
1016                         goto work_done;
1017
1018                 /* Goal is not achieved. We stop process if:
1019
1020                    - if expire reduced to zero. Otherwise, expire is halfed.
1021                    - if table is not full.
1022                    - if we are called from interrupt.
1023                    - jiffies check is just fallback/debug loop breaker.
1024                      We will not spin here for long time in any case.
1025                  */
1026
1027                 RT_CACHE_STAT_INC(gc_goal_miss);
1028
1029                 if (expire == 0)
1030                         break;
1031
1032                 expire >>= 1;
1033 #if RT_CACHE_DEBUG >= 2
1034                 printk(KERN_DEBUG "expire>> %u %d %d %d\n", expire,
1035                                 atomic_read(&ipv4_dst_ops.entries), goal, i);
1036 #endif
1037
1038                 if (atomic_read(&ipv4_dst_ops.entries) < ip_rt_max_size)
1039                         goto out;
1040         } while (!in_softirq() && time_before_eq(jiffies, now));
1041
1042         if (atomic_read(&ipv4_dst_ops.entries) < ip_rt_max_size)
1043                 goto out;
1044         if (net_ratelimit())
1045                 printk(KERN_WARNING "dst cache overflow\n");
1046         RT_CACHE_STAT_INC(gc_dst_overflow);
1047         return 1;
1048
1049 work_done:
1050         expire += ip_rt_gc_min_interval;
1051         if (expire > ip_rt_gc_timeout ||
1052             atomic_read(&ipv4_dst_ops.entries) < ipv4_dst_ops.gc_thresh)
1053                 expire = ip_rt_gc_timeout;
1054 #if RT_CACHE_DEBUG >= 2
1055         printk(KERN_DEBUG "expire++ %u %d %d %d\n", expire,
1056                         atomic_read(&ipv4_dst_ops.entries), goal, rover);
1057 #endif
1058 out:    return 0;
1059 }
1060
1061 /*
1062  * Returns number of entries in a hash chain that have different hash_inputs
1063  */
1064 static int slow_chain_length(const struct rtable *head)
1065 {
1066         int length = 0;
1067         const struct rtable *rth = head;
1068
1069         while (rth) {
1070                 length += has_noalias(head, rth);
1071                 rth = rth->dst.rt_next;
1072         }
1073         return length >> FRACT_BITS;
1074 }
1075
1076 static int rt_intern_hash(unsigned hash, struct rtable *rt,
1077                           struct rtable **rp, struct sk_buff *skb, int ifindex)
1078 {
1079         struct rtable   *rth, **rthp;
1080         unsigned long   now;
1081         struct rtable *cand, **candp;
1082         u32             min_score;
1083         int             chain_length;
1084         int attempts = !in_softirq();
1085
1086 restart:
1087         chain_length = 0;
1088         min_score = ~(u32)0;
1089         cand = NULL;
1090         candp = NULL;
1091         now = jiffies;
1092
1093         if (!rt_caching(dev_net(rt->dst.dev))) {
1094                 /*
1095                  * If we're not caching, just tell the caller we
1096                  * were successful and don't touch the route.  The
1097                  * caller hold the sole reference to the cache entry, and
1098                  * it will be released when the caller is done with it.
1099                  * If we drop it here, the callers have no way to resolve routes
1100                  * when we're not caching.  Instead, just point *rp at rt, so
1101                  * the caller gets a single use out of the route
1102                  * Note that we do rt_free on this new route entry, so that
1103                  * once its refcount hits zero, we are still able to reap it
1104                  * (Thanks Alexey)
1105                  * Note also the rt_free uses call_rcu.  We don't actually
1106                  * need rcu protection here, this is just our path to get
1107                  * on the route gc list.
1108                  */
1109
1110                 rt->dst.flags |= DST_NOCACHE;
1111                 if (rt->rt_type == RTN_UNICAST || rt->fl.iif == 0) {
1112                         int err = arp_bind_neighbour(&rt->dst);
1113                         if (err) {
1114                                 if (net_ratelimit())
1115                                         printk(KERN_WARNING
1116                                             "Neighbour table failure & not caching routes.\n");
1117                                 rt_drop(rt);
1118                                 return err;
1119                         }
1120                 }
1121
1122                 rt_free(rt);
1123                 goto skip_hashing;
1124         }
1125
1126         rthp = &rt_hash_table[hash].chain;
1127
1128         spin_lock_bh(rt_hash_lock_addr(hash));
1129         while ((rth = *rthp) != NULL) {
1130                 if (rt_is_expired(rth)) {
1131                         *rthp = rth->dst.rt_next;
1132                         rt_free(rth);
1133                         continue;
1134                 }
1135                 if (compare_keys(&rth->fl, &rt->fl) && compare_netns(rth, rt)) {
1136                         /* Put it first */
1137                         *rthp = rth->dst.rt_next;
1138                         /*
1139                          * Since lookup is lockfree, the deletion
1140                          * must be visible to another weakly ordered CPU before
1141                          * the insertion at the start of the hash chain.
1142                          */
1143                         rcu_assign_pointer(rth->dst.rt_next,
1144                                            rt_hash_table[hash].chain);
1145                         /*
1146                          * Since lookup is lockfree, the update writes
1147                          * must be ordered for consistency on SMP.
1148                          */
1149                         rcu_assign_pointer(rt_hash_table[hash].chain, rth);
1150
1151                         dst_use(&rth->dst, now);
1152                         spin_unlock_bh(rt_hash_lock_addr(hash));
1153
1154                         rt_drop(rt);
1155                         if (rp)
1156                                 *rp = rth;
1157                         else
1158                                 skb_dst_set(skb, &rth->dst);
1159                         return 0;
1160                 }
1161
1162                 if (!atomic_read(&rth->dst.__refcnt)) {
1163                         u32 score = rt_score(rth);
1164
1165                         if (score <= min_score) {
1166                                 cand = rth;
1167                                 candp = rthp;
1168                                 min_score = score;
1169                         }
1170                 }
1171
1172                 chain_length++;
1173
1174                 rthp = &rth->dst.rt_next;
1175         }
1176
1177         if (cand) {
1178                 /* ip_rt_gc_elasticity used to be average length of chain
1179                  * length, when exceeded gc becomes really aggressive.
1180                  *
1181                  * The second limit is less certain. At the moment it allows
1182                  * only 2 entries per bucket. We will see.
1183                  */
1184                 if (chain_length > ip_rt_gc_elasticity) {
1185                         *candp = cand->dst.rt_next;
1186                         rt_free(cand);
1187                 }
1188         } else {
1189                 if (chain_length > rt_chain_length_max &&
1190                     slow_chain_length(rt_hash_table[hash].chain) > rt_chain_length_max) {
1191                         struct net *net = dev_net(rt->dst.dev);
1192                         int num = ++net->ipv4.current_rt_cache_rebuild_count;
1193                         if (!rt_caching(net)) {
1194                                 printk(KERN_WARNING "%s: %d rebuilds is over limit, route caching disabled\n",
1195                                         rt->dst.dev->name, num);
1196                         }
1197                         rt_emergency_hash_rebuild(net);
1198                         spin_unlock_bh(rt_hash_lock_addr(hash));
1199
1200                         hash = rt_hash(rt->fl.fl4_dst, rt->fl.fl4_src,
1201                                         ifindex, rt_genid(net));
1202                         goto restart;
1203                 }
1204         }
1205
1206         /* Try to bind route to arp only if it is output
1207            route or unicast forwarding path.
1208          */
1209         if (rt->rt_type == RTN_UNICAST || rt->fl.iif == 0) {
1210                 int err = arp_bind_neighbour(&rt->dst);
1211                 if (err) {
1212                         spin_unlock_bh(rt_hash_lock_addr(hash));
1213
1214                         if (err != -ENOBUFS) {
1215                                 rt_drop(rt);
1216                                 return err;
1217                         }
1218
1219                         /* Neighbour tables are full and nothing
1220                            can be released. Try to shrink route cache,
1221                            it is most likely it holds some neighbour records.
1222                          */
1223                         if (attempts-- > 0) {
1224                                 int saved_elasticity = ip_rt_gc_elasticity;
1225                                 int saved_int = ip_rt_gc_min_interval;
1226                                 ip_rt_gc_elasticity     = 1;
1227                                 ip_rt_gc_min_interval   = 0;
1228                                 rt_garbage_collect(&ipv4_dst_ops);
1229                                 ip_rt_gc_min_interval   = saved_int;
1230                                 ip_rt_gc_elasticity     = saved_elasticity;
1231                                 goto restart;
1232                         }
1233
1234                         if (net_ratelimit())
1235                                 printk(KERN_WARNING "ipv4: Neighbour table overflow.\n");
1236                         rt_drop(rt);
1237                         return -ENOBUFS;
1238                 }
1239         }
1240
1241         rt->dst.rt_next = rt_hash_table[hash].chain;
1242
1243 #if RT_CACHE_DEBUG >= 2
1244         if (rt->dst.rt_next) {
1245                 struct rtable *trt;
1246                 printk(KERN_DEBUG "rt_cache @%02x: %pI4",
1247                        hash, &rt->rt_dst);
1248                 for (trt = rt->dst.rt_next; trt; trt = trt->dst.rt_next)
1249                         printk(" . %pI4", &trt->rt_dst);
1250                 printk("\n");
1251         }
1252 #endif
1253         /*
1254          * Since lookup is lockfree, we must make sure
1255          * previous writes to rt are comitted to memory
1256          * before making rt visible to other CPUS.
1257          */
1258         rcu_assign_pointer(rt_hash_table[hash].chain, rt);
1259
1260         spin_unlock_bh(rt_hash_lock_addr(hash));
1261
1262 skip_hashing:
1263         if (rp)
1264                 *rp = rt;
1265         else
1266                 skb_dst_set(skb, &rt->dst);
1267         return 0;
1268 }
1269
1270 void rt_bind_peer(struct rtable *rt, int create)
1271 {
1272         struct inet_peer *peer;
1273
1274         peer = inet_getpeer(rt->rt_dst, create);
1275
1276         if (peer && cmpxchg(&rt->peer, NULL, peer) != NULL)
1277                 inet_putpeer(peer);
1278 }
1279
1280 /*
1281  * Peer allocation may fail only in serious out-of-memory conditions.  However
1282  * we still can generate some output.
1283  * Random ID selection looks a bit dangerous because we have no chances to
1284  * select ID being unique in a reasonable period of time.
1285  * But broken packet identifier may be better than no packet at all.
1286  */
1287 static void ip_select_fb_ident(struct iphdr *iph)
1288 {
1289         static DEFINE_SPINLOCK(ip_fb_id_lock);
1290         static u32 ip_fallback_id;
1291         u32 salt;
1292
1293         spin_lock_bh(&ip_fb_id_lock);
1294         salt = secure_ip_id((__force __be32)ip_fallback_id ^ iph->daddr);
1295         iph->id = htons(salt & 0xFFFF);
1296         ip_fallback_id = salt;
1297         spin_unlock_bh(&ip_fb_id_lock);
1298 }
1299
1300 void __ip_select_ident(struct iphdr *iph, struct dst_entry *dst, int more)
1301 {
1302         struct rtable *rt = (struct rtable *) dst;
1303
1304         if (rt) {
1305                 if (rt->peer == NULL)
1306                         rt_bind_peer(rt, 1);
1307
1308                 /* If peer is attached to destination, it is never detached,
1309                    so that we need not to grab a lock to dereference it.
1310                  */
1311                 if (rt->peer) {
1312                         iph->id = htons(inet_getid(rt->peer, more));
1313                         return;
1314                 }
1315         } else
1316                 printk(KERN_DEBUG "rt_bind_peer(0) @%p\n",
1317                        __builtin_return_address(0));
1318
1319         ip_select_fb_ident(iph);
1320 }
1321 EXPORT_SYMBOL(__ip_select_ident);
1322
1323 static void rt_del(unsigned hash, struct rtable *rt)
1324 {
1325         struct rtable **rthp, *aux;
1326
1327         rthp = &rt_hash_table[hash].chain;
1328         spin_lock_bh(rt_hash_lock_addr(hash));
1329         ip_rt_put(rt);
1330         while ((aux = *rthp) != NULL) {
1331                 if (aux == rt || rt_is_expired(aux)) {
1332                         *rthp = aux->dst.rt_next;
1333                         rt_free(aux);
1334                         continue;
1335                 }
1336                 rthp = &aux->dst.rt_next;
1337         }
1338         spin_unlock_bh(rt_hash_lock_addr(hash));
1339 }
1340
1341 /* called in rcu_read_lock() section */
1342 void ip_rt_redirect(__be32 old_gw, __be32 daddr, __be32 new_gw,
1343                     __be32 saddr, struct net_device *dev)
1344 {
1345         int i, k;
1346         struct in_device *in_dev = __in_dev_get_rcu(dev);
1347         struct rtable *rth, **rthp;
1348         __be32  skeys[2] = { saddr, 0 };
1349         int  ikeys[2] = { dev->ifindex, 0 };
1350         struct netevent_redirect netevent;
1351         struct net *net;
1352
1353         if (!in_dev)
1354                 return;
1355
1356         net = dev_net(dev);
1357         if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
1358             ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
1359             ipv4_is_zeronet(new_gw))
1360                 goto reject_redirect;
1361
1362         if (!rt_caching(net))
1363                 goto reject_redirect;
1364
1365         if (!IN_DEV_SHARED_MEDIA(in_dev)) {
1366                 if (!inet_addr_onlink(in_dev, new_gw, old_gw))
1367                         goto reject_redirect;
1368                 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
1369                         goto reject_redirect;
1370         } else {
1371                 if (inet_addr_type(net, new_gw) != RTN_UNICAST)
1372                         goto reject_redirect;
1373         }
1374
1375         for (i = 0; i < 2; i++) {
1376                 for (k = 0; k < 2; k++) {
1377                         unsigned hash = rt_hash(daddr, skeys[i], ikeys[k],
1378                                                 rt_genid(net));
1379
1380                         rthp=&rt_hash_table[hash].chain;
1381
1382                         while ((rth = rcu_dereference(*rthp)) != NULL) {
1383                                 struct rtable *rt;
1384
1385                                 if (rth->fl.fl4_dst != daddr ||
1386                                     rth->fl.fl4_src != skeys[i] ||
1387                                     rth->fl.oif != ikeys[k] ||
1388                                     rth->fl.iif != 0 ||
1389                                     rt_is_expired(rth) ||
1390                                     !net_eq(dev_net(rth->dst.dev), net)) {
1391                                         rthp = &rth->dst.rt_next;
1392                                         continue;
1393                                 }
1394
1395                                 if (rth->rt_dst != daddr ||
1396                                     rth->rt_src != saddr ||
1397                                     rth->dst.error ||
1398                                     rth->rt_gateway != old_gw ||
1399                                     rth->dst.dev != dev)
1400                                         break;
1401
1402                                 dst_hold(&rth->dst);
1403
1404                                 rt = dst_alloc(&ipv4_dst_ops);
1405                                 if (rt == NULL) {
1406                                         ip_rt_put(rth);
1407                                         return;
1408                                 }
1409
1410                                 /* Copy all the information. */
1411                                 *rt = *rth;
1412                                 rt->dst.__use           = 1;
1413                                 atomic_set(&rt->dst.__refcnt, 1);
1414                                 rt->dst.child           = NULL;
1415                                 if (rt->dst.dev)
1416                                         dev_hold(rt->dst.dev);
1417                                 if (rt->idev)
1418                                         in_dev_hold(rt->idev);
1419                                 rt->dst.obsolete        = -1;
1420                                 rt->dst.lastuse = jiffies;
1421                                 rt->dst.path            = &rt->dst;
1422                                 rt->dst.neighbour       = NULL;
1423                                 rt->dst.hh              = NULL;
1424 #ifdef CONFIG_XFRM
1425                                 rt->dst.xfrm            = NULL;
1426 #endif
1427                                 rt->rt_genid            = rt_genid(net);
1428                                 rt->rt_flags            |= RTCF_REDIRECTED;
1429
1430                                 /* Gateway is different ... */
1431                                 rt->rt_gateway          = new_gw;
1432
1433                                 /* Redirect received -> path was valid */
1434                                 dst_confirm(&rth->dst);
1435
1436                                 if (rt->peer)
1437                                         atomic_inc(&rt->peer->refcnt);
1438
1439                                 if (arp_bind_neighbour(&rt->dst) ||
1440                                     !(rt->dst.neighbour->nud_state &
1441                                             NUD_VALID)) {
1442                                         if (rt->dst.neighbour)
1443                                                 neigh_event_send(rt->dst.neighbour, NULL);
1444                                         ip_rt_put(rth);
1445                                         rt_drop(rt);
1446                                         goto do_next;
1447                                 }
1448
1449                                 netevent.old = &rth->dst;
1450                                 netevent.new = &rt->dst;
1451                                 call_netevent_notifiers(NETEVENT_REDIRECT,
1452                                                         &netevent);
1453
1454                                 rt_del(hash, rth);
1455                                 if (!rt_intern_hash(hash, rt, &rt, NULL, rt->fl.oif))
1456                                         ip_rt_put(rt);
1457                                 goto do_next;
1458                         }
1459                 do_next:
1460                         ;
1461                 }
1462         }
1463         return;
1464
1465 reject_redirect:
1466 #ifdef CONFIG_IP_ROUTE_VERBOSE
1467         if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit())
1468                 printk(KERN_INFO "Redirect from %pI4 on %s about %pI4 ignored.\n"
1469                         "  Advised path = %pI4 -> %pI4\n",
1470                        &old_gw, dev->name, &new_gw,
1471                        &saddr, &daddr);
1472 #endif
1473         ;
1474 }
1475
1476 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
1477 {
1478         struct rtable *rt = (struct rtable *)dst;
1479         struct dst_entry *ret = dst;
1480
1481         if (rt) {
1482                 if (dst->obsolete > 0) {
1483                         ip_rt_put(rt);
1484                         ret = NULL;
1485                 } else if ((rt->rt_flags & RTCF_REDIRECTED) ||
1486                            (rt->dst.expires &&
1487                             time_after_eq(jiffies, rt->dst.expires))) {
1488                         unsigned hash = rt_hash(rt->fl.fl4_dst, rt->fl.fl4_src,
1489                                                 rt->fl.oif,
1490                                                 rt_genid(dev_net(dst->dev)));
1491 #if RT_CACHE_DEBUG >= 1
1492                         printk(KERN_DEBUG "ipv4_negative_advice: redirect to %pI4/%02x dropped\n",
1493                                 &rt->rt_dst, rt->fl.fl4_tos);
1494 #endif
1495                         rt_del(hash, rt);
1496                         ret = NULL;
1497                 }
1498         }
1499         return ret;
1500 }
1501
1502 /*
1503  * Algorithm:
1504  *      1. The first ip_rt_redirect_number redirects are sent
1505  *         with exponential backoff, then we stop sending them at all,
1506  *         assuming that the host ignores our redirects.
1507  *      2. If we did not see packets requiring redirects
1508  *         during ip_rt_redirect_silence, we assume that the host
1509  *         forgot redirected route and start to send redirects again.
1510  *
1511  * This algorithm is much cheaper and more intelligent than dumb load limiting
1512  * in icmp.c.
1513  *
1514  * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
1515  * and "frag. need" (breaks PMTU discovery) in icmp.c.
1516  */
1517
1518 void ip_rt_send_redirect(struct sk_buff *skb)
1519 {
1520         struct rtable *rt = skb_rtable(skb);
1521         struct in_device *in_dev;
1522         int log_martians;
1523
1524         rcu_read_lock();
1525         in_dev = __in_dev_get_rcu(rt->dst.dev);
1526         if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
1527                 rcu_read_unlock();
1528                 return;
1529         }
1530         log_martians = IN_DEV_LOG_MARTIANS(in_dev);
1531         rcu_read_unlock();
1532
1533         /* No redirected packets during ip_rt_redirect_silence;
1534          * reset the algorithm.
1535          */
1536         if (time_after(jiffies, rt->dst.rate_last + ip_rt_redirect_silence))
1537                 rt->dst.rate_tokens = 0;
1538
1539         /* Too many ignored redirects; do not send anything
1540          * set dst.rate_last to the last seen redirected packet.
1541          */
1542         if (rt->dst.rate_tokens >= ip_rt_redirect_number) {
1543                 rt->dst.rate_last = jiffies;
1544                 return;
1545         }
1546
1547         /* Check for load limit; set rate_last to the latest sent
1548          * redirect.
1549          */
1550         if (rt->dst.rate_tokens == 0 ||
1551             time_after(jiffies,
1552                        (rt->dst.rate_last +
1553                         (ip_rt_redirect_load << rt->dst.rate_tokens)))) {
1554                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, rt->rt_gateway);
1555                 rt->dst.rate_last = jiffies;
1556                 ++rt->dst.rate_tokens;
1557 #ifdef CONFIG_IP_ROUTE_VERBOSE
1558                 if (log_martians &&
1559                     rt->dst.rate_tokens == ip_rt_redirect_number &&
1560                     net_ratelimit())
1561                         printk(KERN_WARNING "host %pI4/if%d ignores redirects for %pI4 to %pI4.\n",
1562                                 &rt->rt_src, rt->rt_iif,
1563                                 &rt->rt_dst, &rt->rt_gateway);
1564 #endif
1565         }
1566 }
1567
1568 static int ip_error(struct sk_buff *skb)
1569 {
1570         struct rtable *rt = skb_rtable(skb);
1571         unsigned long now;
1572         int code;
1573
1574         switch (rt->dst.error) {
1575                 case EINVAL:
1576                 default:
1577                         goto out;
1578                 case EHOSTUNREACH:
1579                         code = ICMP_HOST_UNREACH;
1580                         break;
1581                 case ENETUNREACH:
1582                         code = ICMP_NET_UNREACH;
1583                         IP_INC_STATS_BH(dev_net(rt->dst.dev),
1584                                         IPSTATS_MIB_INNOROUTES);
1585                         break;
1586                 case EACCES:
1587                         code = ICMP_PKT_FILTERED;
1588                         break;
1589         }
1590
1591         now = jiffies;
1592         rt->dst.rate_tokens += now - rt->dst.rate_last;
1593         if (rt->dst.rate_tokens > ip_rt_error_burst)
1594                 rt->dst.rate_tokens = ip_rt_error_burst;
1595         rt->dst.rate_last = now;
1596         if (rt->dst.rate_tokens >= ip_rt_error_cost) {
1597                 rt->dst.rate_tokens -= ip_rt_error_cost;
1598                 icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
1599         }
1600
1601 out:    kfree_skb(skb);
1602         return 0;
1603 }
1604
1605 /*
1606  *      The last two values are not from the RFC but
1607  *      are needed for AMPRnet AX.25 paths.
1608  */
1609
1610 static const unsigned short mtu_plateau[] =
1611 {32000, 17914, 8166, 4352, 2002, 1492, 576, 296, 216, 128 };
1612
1613 static inline unsigned short guess_mtu(unsigned short old_mtu)
1614 {
1615         int i;
1616
1617         for (i = 0; i < ARRAY_SIZE(mtu_plateau); i++)
1618                 if (old_mtu > mtu_plateau[i])
1619                         return mtu_plateau[i];
1620         return 68;
1621 }
1622
1623 unsigned short ip_rt_frag_needed(struct net *net, struct iphdr *iph,
1624                                  unsigned short new_mtu,
1625                                  struct net_device *dev)
1626 {
1627         int i, k;
1628         unsigned short old_mtu = ntohs(iph->tot_len);
1629         struct rtable *rth;
1630         int  ikeys[2] = { dev->ifindex, 0 };
1631         __be32  skeys[2] = { iph->saddr, 0, };
1632         __be32  daddr = iph->daddr;
1633         unsigned short est_mtu = 0;
1634
1635         for (k = 0; k < 2; k++) {
1636                 for (i = 0; i < 2; i++) {
1637                         unsigned hash = rt_hash(daddr, skeys[i], ikeys[k],
1638                                                 rt_genid(net));
1639
1640                         rcu_read_lock();
1641                         for (rth = rcu_dereference(rt_hash_table[hash].chain); rth;
1642                              rth = rcu_dereference(rth->dst.rt_next)) {
1643                                 unsigned short mtu = new_mtu;
1644
1645                                 if (rth->fl.fl4_dst != daddr ||
1646                                     rth->fl.fl4_src != skeys[i] ||
1647                                     rth->rt_dst != daddr ||
1648                                     rth->rt_src != iph->saddr ||
1649                                     rth->fl.oif != ikeys[k] ||
1650                                     rth->fl.iif != 0 ||
1651                                     dst_metric_locked(&rth->dst, RTAX_MTU) ||
1652                                     !net_eq(dev_net(rth->dst.dev), net) ||
1653                                     rt_is_expired(rth))
1654                                         continue;
1655
1656                                 if (new_mtu < 68 || new_mtu >= old_mtu) {
1657
1658                                         /* BSD 4.2 compatibility hack :-( */
1659                                         if (mtu == 0 &&
1660                                             old_mtu >= dst_mtu(&rth->dst) &&
1661                                             old_mtu >= 68 + (iph->ihl << 2))
1662                                                 old_mtu -= iph->ihl << 2;
1663
1664                                         mtu = guess_mtu(old_mtu);
1665                                 }
1666                                 if (mtu <= dst_mtu(&rth->dst)) {
1667                                         if (mtu < dst_mtu(&rth->dst)) {
1668                                                 dst_confirm(&rth->dst);
1669                                                 if (mtu < ip_rt_min_pmtu) {
1670                                                         mtu = ip_rt_min_pmtu;
1671                                                         rth->dst.metrics[RTAX_LOCK-1] |=
1672                                                                 (1 << RTAX_MTU);
1673                                                 }
1674                                                 rth->dst.metrics[RTAX_MTU-1] = mtu;
1675                                                 dst_set_expires(&rth->dst,
1676                                                         ip_rt_mtu_expires);
1677                                         }
1678                                         est_mtu = mtu;
1679                                 }
1680                         }
1681                         rcu_read_unlock();
1682                 }
1683         }
1684         return est_mtu ? : new_mtu;
1685 }
1686
1687 static void ip_rt_update_pmtu(struct dst_entry *dst, u32 mtu)
1688 {
1689         if (dst_mtu(dst) > mtu && mtu >= 68 &&
1690             !(dst_metric_locked(dst, RTAX_MTU))) {
1691                 if (mtu < ip_rt_min_pmtu) {
1692                         mtu = ip_rt_min_pmtu;
1693                         dst->metrics[RTAX_LOCK-1] |= (1 << RTAX_MTU);
1694                 }
1695                 dst->metrics[RTAX_MTU-1] = mtu;
1696                 dst_set_expires(dst, ip_rt_mtu_expires);
1697                 call_netevent_notifiers(NETEVENT_PMTU_UPDATE, dst);
1698         }
1699 }
1700
1701 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
1702 {
1703         if (rt_is_expired((struct rtable *)dst))
1704                 return NULL;
1705         return dst;
1706 }
1707
1708 static void ipv4_dst_destroy(struct dst_entry *dst)
1709 {
1710         struct rtable *rt = (struct rtable *) dst;
1711         struct inet_peer *peer = rt->peer;
1712         struct in_device *idev = rt->idev;
1713
1714         if (peer) {
1715                 rt->peer = NULL;
1716                 inet_putpeer(peer);
1717         }
1718
1719         if (idev) {
1720                 rt->idev = NULL;
1721                 in_dev_put(idev);
1722         }
1723 }
1724
1725 static void ipv4_dst_ifdown(struct dst_entry *dst, struct net_device *dev,
1726                             int how)
1727 {
1728         struct rtable *rt = (struct rtable *) dst;
1729         struct in_device *idev = rt->idev;
1730         if (dev != dev_net(dev)->loopback_dev && idev && idev->dev == dev) {
1731                 struct in_device *loopback_idev =
1732                         in_dev_get(dev_net(dev)->loopback_dev);
1733                 if (loopback_idev) {
1734                         rt->idev = loopback_idev;
1735                         in_dev_put(idev);
1736                 }
1737         }
1738 }
1739
1740 static void ipv4_link_failure(struct sk_buff *skb)
1741 {
1742         struct rtable *rt;
1743
1744         icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
1745
1746         rt = skb_rtable(skb);
1747         if (rt)
1748                 dst_set_expires(&rt->dst, 0);
1749 }
1750
1751 static int ip_rt_bug(struct sk_buff *skb)
1752 {
1753         printk(KERN_DEBUG "ip_rt_bug: %pI4 -> %pI4, %s\n",
1754                 &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
1755                 skb->dev ? skb->dev->name : "?");
1756         kfree_skb(skb);
1757         return 0;
1758 }
1759
1760 /*
1761    We do not cache source address of outgoing interface,
1762    because it is used only by IP RR, TS and SRR options,
1763    so that it out of fast path.
1764
1765    BTW remember: "addr" is allowed to be not aligned
1766    in IP options!
1767  */
1768
1769 void ip_rt_get_source(u8 *addr, struct rtable *rt)
1770 {
1771         __be32 src;
1772         struct fib_result res;
1773
1774         if (rt->fl.iif == 0)
1775                 src = rt->rt_src;
1776         else {
1777                 rcu_read_lock();
1778                 if (fib_lookup(dev_net(rt->dst.dev), &rt->fl, &res) == 0)
1779                         src = FIB_RES_PREFSRC(res);
1780                 else
1781                         src = inet_select_addr(rt->dst.dev, rt->rt_gateway,
1782                                         RT_SCOPE_UNIVERSE);
1783                 rcu_read_unlock();
1784         }
1785         memcpy(addr, &src, 4);
1786 }
1787
1788 #ifdef CONFIG_NET_CLS_ROUTE
1789 static void set_class_tag(struct rtable *rt, u32 tag)
1790 {
1791         if (!(rt->dst.tclassid & 0xFFFF))
1792                 rt->dst.tclassid |= tag & 0xFFFF;
1793         if (!(rt->dst.tclassid & 0xFFFF0000))
1794                 rt->dst.tclassid |= tag & 0xFFFF0000;
1795 }
1796 #endif
1797
1798 static void rt_set_nexthop(struct rtable *rt, struct fib_result *res, u32 itag)
1799 {
1800         struct fib_info *fi = res->fi;
1801
1802         if (fi) {
1803                 if (FIB_RES_GW(*res) &&
1804                     FIB_RES_NH(*res).nh_scope == RT_SCOPE_LINK)
1805                         rt->rt_gateway = FIB_RES_GW(*res);
1806                 memcpy(rt->dst.metrics, fi->fib_metrics,
1807                        sizeof(rt->dst.metrics));
1808                 if (fi->fib_mtu == 0) {
1809                         rt->dst.metrics[RTAX_MTU-1] = rt->dst.dev->mtu;
1810                         if (dst_metric_locked(&rt->dst, RTAX_MTU) &&
1811                             rt->rt_gateway != rt->rt_dst &&
1812                             rt->dst.dev->mtu > 576)
1813                                 rt->dst.metrics[RTAX_MTU-1] = 576;
1814                 }
1815 #ifdef CONFIG_NET_CLS_ROUTE
1816                 rt->dst.tclassid = FIB_RES_NH(*res).nh_tclassid;
1817 #endif
1818         } else
1819                 rt->dst.metrics[RTAX_MTU-1]= rt->dst.dev->mtu;
1820
1821         if (dst_metric(&rt->dst, RTAX_HOPLIMIT) == 0)
1822                 rt->dst.metrics[RTAX_HOPLIMIT-1] = sysctl_ip_default_ttl;
1823         if (dst_mtu(&rt->dst) > IP_MAX_MTU)
1824                 rt->dst.metrics[RTAX_MTU-1] = IP_MAX_MTU;
1825         if (dst_metric(&rt->dst, RTAX_ADVMSS) == 0)
1826                 rt->dst.metrics[RTAX_ADVMSS-1] = max_t(unsigned int, rt->dst.dev->mtu - 40,
1827                                        ip_rt_min_advmss);
1828         if (dst_metric(&rt->dst, RTAX_ADVMSS) > 65535 - 40)
1829                 rt->dst.metrics[RTAX_ADVMSS-1] = 65535 - 40;
1830
1831 #ifdef CONFIG_NET_CLS_ROUTE
1832 #ifdef CONFIG_IP_MULTIPLE_TABLES
1833         set_class_tag(rt, fib_rules_tclass(res));
1834 #endif
1835         set_class_tag(rt, itag);
1836 #endif
1837         rt->rt_type = res->type;
1838 }
1839
1840 /* called in rcu_read_lock() section */
1841 static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1842                                 u8 tos, struct net_device *dev, int our)
1843 {
1844         unsigned int hash;
1845         struct rtable *rth;
1846         __be32 spec_dst;
1847         struct in_device *in_dev = __in_dev_get_rcu(dev);
1848         u32 itag = 0;
1849         int err;
1850
1851         /* Primary sanity checks. */
1852
1853         if (in_dev == NULL)
1854                 return -EINVAL;
1855
1856         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1857             ipv4_is_loopback(saddr) || skb->protocol != htons(ETH_P_IP))
1858                 goto e_inval;
1859
1860         if (ipv4_is_zeronet(saddr)) {
1861                 if (!ipv4_is_local_multicast(daddr))
1862                         goto e_inval;
1863                 spec_dst = inet_select_addr(dev, 0, RT_SCOPE_LINK);
1864         } else {
1865                 err = fib_validate_source(saddr, 0, tos, 0, dev, &spec_dst,
1866                                           &itag, 0);
1867                 if (err < 0)
1868                         goto e_err;
1869         }
1870         rth = dst_alloc(&ipv4_dst_ops);
1871         if (!rth)
1872                 goto e_nobufs;
1873
1874         rth->dst.output = ip_rt_bug;
1875         rth->dst.obsolete = -1;
1876
1877         atomic_set(&rth->dst.__refcnt, 1);
1878         rth->dst.flags= DST_HOST;
1879         if (IN_DEV_CONF_GET(in_dev, NOPOLICY))
1880                 rth->dst.flags |= DST_NOPOLICY;
1881         rth->fl.fl4_dst = daddr;
1882         rth->rt_dst     = daddr;
1883         rth->fl.fl4_tos = tos;
1884         rth->fl.mark    = skb->mark;
1885         rth->fl.fl4_src = saddr;
1886         rth->rt_src     = saddr;
1887 #ifdef CONFIG_NET_CLS_ROUTE
1888         rth->dst.tclassid = itag;
1889 #endif
1890         rth->rt_iif     =
1891         rth->fl.iif     = dev->ifindex;
1892         rth->dst.dev    = init_net.loopback_dev;
1893         dev_hold(rth->dst.dev);
1894         rth->idev       = in_dev_get(rth->dst.dev);
1895         rth->fl.oif     = 0;
1896         rth->rt_gateway = daddr;
1897         rth->rt_spec_dst= spec_dst;
1898         rth->rt_genid   = rt_genid(dev_net(dev));
1899         rth->rt_flags   = RTCF_MULTICAST;
1900         rth->rt_type    = RTN_MULTICAST;
1901         if (our) {
1902                 rth->dst.input= ip_local_deliver;
1903                 rth->rt_flags |= RTCF_LOCAL;
1904         }
1905
1906 #ifdef CONFIG_IP_MROUTE
1907         if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
1908                 rth->dst.input = ip_mr_input;
1909 #endif
1910         RT_CACHE_STAT_INC(in_slow_mc);
1911
1912         hash = rt_hash(daddr, saddr, dev->ifindex, rt_genid(dev_net(dev)));
1913         return rt_intern_hash(hash, rth, NULL, skb, dev->ifindex);
1914
1915 e_nobufs:
1916         return -ENOBUFS;
1917 e_inval:
1918         return -EINVAL;
1919 e_err:
1920         return err;
1921 }
1922
1923
1924 static void ip_handle_martian_source(struct net_device *dev,
1925                                      struct in_device *in_dev,
1926                                      struct sk_buff *skb,
1927                                      __be32 daddr,
1928                                      __be32 saddr)
1929 {
1930         RT_CACHE_STAT_INC(in_martian_src);
1931 #ifdef CONFIG_IP_ROUTE_VERBOSE
1932         if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
1933                 /*
1934                  *      RFC1812 recommendation, if source is martian,
1935                  *      the only hint is MAC header.
1936                  */
1937                 printk(KERN_WARNING "martian source %pI4 from %pI4, on dev %s\n",
1938                         &daddr, &saddr, dev->name);
1939                 if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
1940                         int i;
1941                         const unsigned char *p = skb_mac_header(skb);
1942                         printk(KERN_WARNING "ll header: ");
1943                         for (i = 0; i < dev->hard_header_len; i++, p++) {
1944                                 printk("%02x", *p);
1945                                 if (i < (dev->hard_header_len - 1))
1946                                         printk(":");
1947                         }
1948                         printk("\n");
1949                 }
1950         }
1951 #endif
1952 }
1953
1954 /* called in rcu_read_lock() section */
1955 static int __mkroute_input(struct sk_buff *skb,
1956                            struct fib_result *res,
1957                            struct in_device *in_dev,
1958                            __be32 daddr, __be32 saddr, u32 tos,
1959                            struct rtable **result)
1960 {
1961         struct rtable *rth;
1962         int err;
1963         struct in_device *out_dev;
1964         unsigned int flags = 0;
1965         __be32 spec_dst;
1966         u32 itag;
1967
1968         /* get a working reference to the output device */
1969         out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
1970         if (out_dev == NULL) {
1971                 if (net_ratelimit())
1972                         printk(KERN_CRIT "Bug in ip_route_input" \
1973                                "_slow(). Please, report\n");
1974                 return -EINVAL;
1975         }
1976
1977
1978         err = fib_validate_source(saddr, daddr, tos, FIB_RES_OIF(*res),
1979                                   in_dev->dev, &spec_dst, &itag, skb->mark);
1980         if (err < 0) {
1981                 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
1982                                          saddr);
1983
1984                 goto cleanup;
1985         }
1986
1987         if (err)
1988                 flags |= RTCF_DIRECTSRC;
1989
1990         if (out_dev == in_dev && err &&
1991             (IN_DEV_SHARED_MEDIA(out_dev) ||
1992              inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res))))
1993                 flags |= RTCF_DOREDIRECT;
1994
1995         if (skb->protocol != htons(ETH_P_IP)) {
1996                 /* Not IP (i.e. ARP). Do not create route, if it is
1997                  * invalid for proxy arp. DNAT routes are always valid.
1998                  *
1999                  * Proxy arp feature have been extended to allow, ARP
2000                  * replies back to the same interface, to support
2001                  * Private VLAN switch technologies. See arp.c.
2002                  */
2003                 if (out_dev == in_dev &&
2004                     IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
2005                         err = -EINVAL;
2006                         goto cleanup;
2007                 }
2008         }
2009
2010
2011         rth = dst_alloc(&ipv4_dst_ops);
2012         if (!rth) {
2013                 err = -ENOBUFS;
2014                 goto cleanup;
2015         }
2016
2017         atomic_set(&rth->dst.__refcnt, 1);
2018         rth->dst.flags= DST_HOST;
2019         if (IN_DEV_CONF_GET(in_dev, NOPOLICY))
2020                 rth->dst.flags |= DST_NOPOLICY;
2021         if (IN_DEV_CONF_GET(out_dev, NOXFRM))
2022                 rth->dst.flags |= DST_NOXFRM;
2023         rth->fl.fl4_dst = daddr;
2024         rth->rt_dst     = daddr;
2025         rth->fl.fl4_tos = tos;
2026         rth->fl.mark    = skb->mark;
2027         rth->fl.fl4_src = saddr;
2028         rth->rt_src     = saddr;
2029         rth->rt_gateway = daddr;
2030         rth->rt_iif     =
2031                 rth->fl.iif     = in_dev->dev->ifindex;
2032         rth->dst.dev    = (out_dev)->dev;
2033         dev_hold(rth->dst.dev);
2034         rth->idev       = in_dev_get(rth->dst.dev);
2035         rth->fl.oif     = 0;
2036         rth->rt_spec_dst= spec_dst;
2037
2038         rth->dst.obsolete = -1;
2039         rth->dst.input = ip_forward;
2040         rth->dst.output = ip_output;
2041         rth->rt_genid = rt_genid(dev_net(rth->dst.dev));
2042
2043         rt_set_nexthop(rth, res, itag);
2044
2045         rth->rt_flags = flags;
2046
2047         *result = rth;
2048         err = 0;
2049  cleanup:
2050         return err;
2051 }
2052
2053 static int ip_mkroute_input(struct sk_buff *skb,
2054                             struct fib_result *res,
2055                             const struct flowi *fl,
2056                             struct in_device *in_dev,
2057                             __be32 daddr, __be32 saddr, u32 tos)
2058 {
2059         struct rtable* rth = NULL;
2060         int err;
2061         unsigned hash;
2062
2063 #ifdef CONFIG_IP_ROUTE_MULTIPATH
2064         if (res->fi && res->fi->fib_nhs > 1 && fl->oif == 0)
2065                 fib_select_multipath(fl, res);
2066 #endif
2067
2068         /* create a routing cache entry */
2069         err = __mkroute_input(skb, res, in_dev, daddr, saddr, tos, &rth);
2070         if (err)
2071                 return err;
2072
2073         /* put it into the cache */
2074         hash = rt_hash(daddr, saddr, fl->iif,
2075                        rt_genid(dev_net(rth->dst.dev)));
2076         return rt_intern_hash(hash, rth, NULL, skb, fl->iif);
2077 }
2078
2079 /*
2080  *      NOTE. We drop all the packets that has local source
2081  *      addresses, because every properly looped back packet
2082  *      must have correct destination already attached by output routine.
2083  *
2084  *      Such approach solves two big problems:
2085  *      1. Not simplex devices are handled properly.
2086  *      2. IP spoofing attempts are filtered with 100% of guarantee.
2087  *      called with rcu_read_lock()
2088  */
2089
2090 static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2091                                u8 tos, struct net_device *dev)
2092 {
2093         struct fib_result res;
2094         struct in_device *in_dev = __in_dev_get_rcu(dev);
2095         struct flowi fl = { .nl_u = { .ip4_u =
2096                                       { .daddr = daddr,
2097                                         .saddr = saddr,
2098                                         .tos = tos,
2099                                         .scope = RT_SCOPE_UNIVERSE,
2100                                       } },
2101                             .mark = skb->mark,
2102                             .iif = dev->ifindex };
2103         unsigned        flags = 0;
2104         u32             itag = 0;
2105         struct rtable * rth;
2106         unsigned        hash;
2107         __be32          spec_dst;
2108         int             err = -EINVAL;
2109         struct net    * net = dev_net(dev);
2110
2111         /* IP on this device is disabled. */
2112
2113         if (!in_dev)
2114                 goto out;
2115
2116         /* Check for the most weird martians, which can be not detected
2117            by fib_lookup.
2118          */
2119
2120         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
2121             ipv4_is_loopback(saddr))
2122                 goto martian_source;
2123
2124         if (daddr == htonl(0xFFFFFFFF) || (saddr == 0 && daddr == 0))
2125                 goto brd_input;
2126
2127         /* Accept zero addresses only to limited broadcast;
2128          * I even do not know to fix it or not. Waiting for complains :-)
2129          */
2130         if (ipv4_is_zeronet(saddr))
2131                 goto martian_source;
2132
2133         if (ipv4_is_lbcast(daddr) || ipv4_is_zeronet(daddr) ||
2134             ipv4_is_loopback(daddr))
2135                 goto martian_destination;
2136
2137         /*
2138          *      Now we are ready to route packet.
2139          */
2140         err = fib_lookup(net, &fl, &res);
2141         if (err != 0) {
2142                 if (!IN_DEV_FORWARD(in_dev))
2143                         goto e_hostunreach;
2144                 goto no_route;
2145         }
2146
2147         RT_CACHE_STAT_INC(in_slow_tot);
2148
2149         if (res.type == RTN_BROADCAST)
2150                 goto brd_input;
2151
2152         if (res.type == RTN_LOCAL) {
2153                 err = fib_validate_source(saddr, daddr, tos,
2154                                           net->loopback_dev->ifindex,
2155                                           dev, &spec_dst, &itag, skb->mark);
2156                 if (err < 0)
2157                         goto martian_source_keep_err;
2158                 if (err)
2159                         flags |= RTCF_DIRECTSRC;
2160                 spec_dst = daddr;
2161                 goto local_input;
2162         }
2163
2164         if (!IN_DEV_FORWARD(in_dev))
2165                 goto e_hostunreach;
2166         if (res.type != RTN_UNICAST)
2167                 goto martian_destination;
2168
2169         err = ip_mkroute_input(skb, &res, &fl, in_dev, daddr, saddr, tos);
2170 out:    return err;
2171
2172 brd_input:
2173         if (skb->protocol != htons(ETH_P_IP))
2174                 goto e_inval;
2175
2176         if (ipv4_is_zeronet(saddr))
2177                 spec_dst = inet_select_addr(dev, 0, RT_SCOPE_LINK);
2178         else {
2179                 err = fib_validate_source(saddr, 0, tos, 0, dev, &spec_dst,
2180                                           &itag, skb->mark);
2181                 if (err < 0)
2182                         goto martian_source_keep_err;
2183                 if (err)
2184                         flags |= RTCF_DIRECTSRC;
2185         }
2186         flags |= RTCF_BROADCAST;
2187         res.type = RTN_BROADCAST;
2188         RT_CACHE_STAT_INC(in_brd);
2189
2190 local_input:
2191         rth = dst_alloc(&ipv4_dst_ops);
2192         if (!rth)
2193                 goto e_nobufs;
2194
2195         rth->dst.output= ip_rt_bug;
2196         rth->dst.obsolete = -1;
2197         rth->rt_genid = rt_genid(net);
2198
2199         atomic_set(&rth->dst.__refcnt, 1);
2200         rth->dst.flags= DST_HOST;
2201         if (IN_DEV_CONF_GET(in_dev, NOPOLICY))
2202                 rth->dst.flags |= DST_NOPOLICY;
2203         rth->fl.fl4_dst = daddr;
2204         rth->rt_dst     = daddr;
2205         rth->fl.fl4_tos = tos;
2206         rth->fl.mark    = skb->mark;
2207         rth->fl.fl4_src = saddr;
2208         rth->rt_src     = saddr;
2209 #ifdef CONFIG_NET_CLS_ROUTE
2210         rth->dst.tclassid = itag;
2211 #endif
2212         rth->rt_iif     =
2213         rth->fl.iif     = dev->ifindex;
2214         rth->dst.dev    = net->loopback_dev;
2215         dev_hold(rth->dst.dev);
2216         rth->idev       = in_dev_get(rth->dst.dev);
2217         rth->rt_gateway = daddr;
2218         rth->rt_spec_dst= spec_dst;
2219         rth->dst.input= ip_local_deliver;
2220         rth->rt_flags   = flags|RTCF_LOCAL;
2221         if (res.type == RTN_UNREACHABLE) {
2222                 rth->dst.input= ip_error;
2223                 rth->dst.error= -err;
2224                 rth->rt_flags   &= ~RTCF_LOCAL;
2225         }
2226         rth->rt_type    = res.type;
2227         hash = rt_hash(daddr, saddr, fl.iif, rt_genid(net));
2228         err = rt_intern_hash(hash, rth, NULL, skb, fl.iif);
2229         goto out;
2230
2231 no_route:
2232         RT_CACHE_STAT_INC(in_no_route);
2233         spec_dst = inet_select_addr(dev, 0, RT_SCOPE_UNIVERSE);
2234         res.type = RTN_UNREACHABLE;
2235         if (err == -ESRCH)
2236                 err = -ENETUNREACH;
2237         goto local_input;
2238
2239         /*
2240          *      Do not cache martian addresses: they should be logged (RFC1812)
2241          */
2242 martian_destination:
2243         RT_CACHE_STAT_INC(in_martian_dst);
2244 #ifdef CONFIG_IP_ROUTE_VERBOSE
2245         if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit())
2246                 printk(KERN_WARNING "martian destination %pI4 from %pI4, dev %s\n",
2247                         &daddr, &saddr, dev->name);
2248 #endif
2249
2250 e_hostunreach:
2251         err = -EHOSTUNREACH;
2252         goto out;
2253
2254 e_inval:
2255         err = -EINVAL;
2256         goto out;
2257
2258 e_nobufs:
2259         err = -ENOBUFS;
2260         goto out;
2261
2262 martian_source:
2263         err = -EINVAL;
2264 martian_source_keep_err:
2265         ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
2266         goto out;
2267 }
2268
2269 int ip_route_input_common(struct sk_buff *skb, __be32 daddr, __be32 saddr,
2270                            u8 tos, struct net_device *dev, bool noref)
2271 {
2272         struct rtable * rth;
2273         unsigned        hash;
2274         int iif = dev->ifindex;
2275         struct net *net;
2276         int res;
2277
2278         net = dev_net(dev);
2279
2280         rcu_read_lock();
2281
2282         if (!rt_caching(net))
2283                 goto skip_cache;
2284
2285         tos &= IPTOS_RT_MASK;
2286         hash = rt_hash(daddr, saddr, iif, rt_genid(net));
2287
2288         for (rth = rcu_dereference(rt_hash_table[hash].chain); rth;
2289              rth = rcu_dereference(rth->dst.rt_next)) {
2290                 if ((((__force u32)rth->fl.fl4_dst ^ (__force u32)daddr) |
2291                      ((__force u32)rth->fl.fl4_src ^ (__force u32)saddr) |
2292                      (rth->fl.iif ^ iif) |
2293                      rth->fl.oif |
2294                      (rth->fl.fl4_tos ^ tos)) == 0 &&
2295                     rth->fl.mark == skb->mark &&
2296                     net_eq(dev_net(rth->dst.dev), net) &&
2297                     !rt_is_expired(rth)) {
2298                         if (noref) {
2299                                 dst_use_noref(&rth->dst, jiffies);
2300                                 skb_dst_set_noref(skb, &rth->dst);
2301                         } else {
2302                                 dst_use(&rth->dst, jiffies);
2303                                 skb_dst_set(skb, &rth->dst);
2304                         }
2305                         RT_CACHE_STAT_INC(in_hit);
2306                         rcu_read_unlock();
2307                         return 0;
2308                 }
2309                 RT_CACHE_STAT_INC(in_hlist_search);
2310         }
2311
2312 skip_cache:
2313         /* Multicast recognition logic is moved from route cache to here.
2314            The problem was that too many Ethernet cards have broken/missing
2315            hardware multicast filters :-( As result the host on multicasting
2316            network acquires a lot of useless route cache entries, sort of
2317            SDR messages from all the world. Now we try to get rid of them.
2318            Really, provided software IP multicast filter is organized
2319            reasonably (at least, hashed), it does not result in a slowdown
2320            comparing with route cache reject entries.
2321            Note, that multicast routers are not affected, because
2322            route cache entry is created eventually.
2323          */
2324         if (ipv4_is_multicast(daddr)) {
2325                 struct in_device *in_dev = __in_dev_get_rcu(dev);
2326
2327                 if (in_dev) {
2328                         int our = ip_check_mc(in_dev, daddr, saddr,
2329                                               ip_hdr(skb)->protocol);
2330                         if (our
2331 #ifdef CONFIG_IP_MROUTE
2332                                 ||
2333                             (!ipv4_is_local_multicast(daddr) &&
2334                              IN_DEV_MFORWARD(in_dev))
2335 #endif
2336                            ) {
2337                                 int res = ip_route_input_mc(skb, daddr, saddr,
2338                                                             tos, dev, our);
2339                                 rcu_read_unlock();
2340                                 return res;
2341                         }
2342                 }
2343                 rcu_read_unlock();
2344                 return -EINVAL;
2345         }
2346         res = ip_route_input_slow(skb, daddr, saddr, tos, dev);
2347         rcu_read_unlock();
2348         return res;
2349 }
2350 EXPORT_SYMBOL(ip_route_input_common);
2351
2352 /* called with rcu_read_lock() */
2353 static int __mkroute_output(struct rtable **result,
2354                             struct fib_result *res,
2355                             const struct flowi *fl,
2356                             const struct flowi *oldflp,
2357                             struct net_device *dev_out,
2358                             unsigned flags)
2359 {
2360         struct rtable *rth;
2361         struct in_device *in_dev;
2362         u32 tos = RT_FL_TOS(oldflp);
2363
2364         if (ipv4_is_loopback(fl->fl4_src) && !(dev_out->flags & IFF_LOOPBACK))
2365                 return -EINVAL;
2366
2367         if (fl->fl4_dst == htonl(0xFFFFFFFF))
2368                 res->type = RTN_BROADCAST;
2369         else if (ipv4_is_multicast(fl->fl4_dst))
2370                 res->type = RTN_MULTICAST;
2371         else if (ipv4_is_lbcast(fl->fl4_dst) || ipv4_is_zeronet(fl->fl4_dst))
2372                 return -EINVAL;
2373
2374         if (dev_out->flags & IFF_LOOPBACK)
2375                 flags |= RTCF_LOCAL;
2376
2377         in_dev = __in_dev_get_rcu(dev_out);
2378         if (!in_dev)
2379                 return -EINVAL;
2380
2381         if (res->type == RTN_BROADCAST) {
2382                 flags |= RTCF_BROADCAST | RTCF_LOCAL;
2383                 res->fi = NULL;
2384         } else if (res->type == RTN_MULTICAST) {
2385                 flags |= RTCF_MULTICAST | RTCF_LOCAL;
2386                 if (!ip_check_mc(in_dev, oldflp->fl4_dst, oldflp->fl4_src,
2387                                  oldflp->proto))
2388                         flags &= ~RTCF_LOCAL;
2389                 /* If multicast route do not exist use
2390                  * default one, but do not gateway in this case.
2391                  * Yes, it is hack.
2392                  */
2393                 if (res->fi && res->prefixlen < 4)
2394                         res->fi = NULL;
2395         }
2396
2397
2398         rth = dst_alloc(&ipv4_dst_ops);
2399         if (!rth)
2400                 return -ENOBUFS;
2401
2402         in_dev_hold(in_dev);
2403         rth->idev = in_dev;
2404
2405         atomic_set(&rth->dst.__refcnt, 1);
2406         rth->dst.flags= DST_HOST;
2407         if (IN_DEV_CONF_GET(in_dev, NOXFRM))
2408                 rth->dst.flags |= DST_NOXFRM;
2409         if (IN_DEV_CONF_GET(in_dev, NOPOLICY))
2410                 rth->dst.flags |= DST_NOPOLICY;
2411
2412         rth->fl.fl4_dst = oldflp->fl4_dst;
2413         rth->fl.fl4_tos = tos;
2414         rth->fl.fl4_src = oldflp->fl4_src;
2415         rth->fl.oif     = oldflp->oif;
2416         rth->fl.mark    = oldflp->mark;
2417         rth->rt_dst     = fl->fl4_dst;
2418         rth->rt_src     = fl->fl4_src;
2419         rth->rt_iif     = oldflp->oif ? : dev_out->ifindex;
2420         /* get references to the devices that are to be hold by the routing
2421            cache entry */
2422         rth->dst.dev    = dev_out;
2423         dev_hold(dev_out);
2424         rth->rt_gateway = fl->fl4_dst;
2425         rth->rt_spec_dst= fl->fl4_src;
2426
2427         rth->dst.output=ip_output;
2428         rth->dst.obsolete = -1;
2429         rth->rt_genid = rt_genid(dev_net(dev_out));
2430
2431         RT_CACHE_STAT_INC(out_slow_tot);
2432
2433         if (flags & RTCF_LOCAL) {
2434                 rth->dst.input = ip_local_deliver;
2435                 rth->rt_spec_dst = fl->fl4_dst;
2436         }
2437         if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
2438                 rth->rt_spec_dst = fl->fl4_src;
2439                 if (flags & RTCF_LOCAL &&
2440                     !(dev_out->flags & IFF_LOOPBACK)) {
2441                         rth->dst.output = ip_mc_output;
2442                         RT_CACHE_STAT_INC(out_slow_mc);
2443                 }
2444 #ifdef CONFIG_IP_MROUTE
2445                 if (res->type == RTN_MULTICAST) {
2446                         if (IN_DEV_MFORWARD(in_dev) &&
2447                             !ipv4_is_local_multicast(oldflp->fl4_dst)) {
2448                                 rth->dst.input = ip_mr_input;
2449                                 rth->dst.output = ip_mc_output;
2450                         }
2451                 }
2452 #endif
2453         }
2454
2455         rt_set_nexthop(rth, res, 0);
2456
2457         rth->rt_flags = flags;
2458         *result = rth;
2459         return 0;
2460 }
2461
2462 /* called with rcu_read_lock() */
2463 static int ip_mkroute_output(struct rtable **rp,
2464                              struct fib_result *res,
2465                              const struct flowi *fl,
2466                              const struct flowi *oldflp,
2467                              struct net_device *dev_out,
2468                              unsigned flags)
2469 {
2470         struct rtable *rth = NULL;
2471         int err = __mkroute_output(&rth, res, fl, oldflp, dev_out, flags);
2472         unsigned hash;
2473         if (err == 0) {
2474                 hash = rt_hash(oldflp->fl4_dst, oldflp->fl4_src, oldflp->oif,
2475                                rt_genid(dev_net(dev_out)));
2476                 err = rt_intern_hash(hash, rth, rp, NULL, oldflp->oif);
2477         }
2478
2479         return err;
2480 }
2481
2482 /*
2483  * Major route resolver routine.
2484  * called with rcu_read_lock();
2485  */
2486
2487 static int ip_route_output_slow(struct net *net, struct rtable **rp,
2488                                 const struct flowi *oldflp)
2489 {
2490         u32 tos = RT_FL_TOS(oldflp);
2491         struct flowi fl = { .nl_u = { .ip4_u =
2492                                       { .daddr = oldflp->fl4_dst,
2493                                         .saddr = oldflp->fl4_src,
2494                                         .tos = tos & IPTOS_RT_MASK,
2495                                         .scope = ((tos & RTO_ONLINK) ?
2496                                                   RT_SCOPE_LINK :
2497                                                   RT_SCOPE_UNIVERSE),
2498                                       } },
2499                             .mark = oldflp->mark,
2500                             .iif = net->loopback_dev->ifindex,
2501                             .oif = oldflp->oif };
2502         struct fib_result res;
2503         unsigned int flags = 0;
2504         struct net_device *dev_out = NULL;
2505         int err;
2506
2507
2508         res.fi          = NULL;
2509 #ifdef CONFIG_IP_MULTIPLE_TABLES
2510         res.r           = NULL;
2511 #endif
2512
2513         if (oldflp->fl4_src) {
2514                 err = -EINVAL;
2515                 if (ipv4_is_multicast(oldflp->fl4_src) ||
2516                     ipv4_is_lbcast(oldflp->fl4_src) ||
2517                     ipv4_is_zeronet(oldflp->fl4_src))
2518                         goto out;
2519
2520                 /* I removed check for oif == dev_out->oif here.
2521                    It was wrong for two reasons:
2522                    1. ip_dev_find(net, saddr) can return wrong iface, if saddr
2523                       is assigned to multiple interfaces.
2524                    2. Moreover, we are allowed to send packets with saddr
2525                       of another iface. --ANK
2526                  */
2527
2528                 if (oldflp->oif == 0 &&
2529                     (ipv4_is_multicast(oldflp->fl4_dst) ||
2530                      oldflp->fl4_dst == htonl(0xFFFFFFFF))) {
2531                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2532                         dev_out = __ip_dev_find(net, oldflp->fl4_src, false);
2533                         if (dev_out == NULL)
2534                                 goto out;
2535
2536                         /* Special hack: user can direct multicasts
2537                            and limited broadcast via necessary interface
2538                            without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
2539                            This hack is not just for fun, it allows
2540                            vic,vat and friends to work.
2541                            They bind socket to loopback, set ttl to zero
2542                            and expect that it will work.
2543                            From the viewpoint of routing cache they are broken,
2544                            because we are not allowed to build multicast path
2545                            with loopback source addr (look, routing cache
2546                            cannot know, that ttl is zero, so that packet
2547                            will not leave this host and route is valid).
2548                            Luckily, this hack is good workaround.
2549                          */
2550
2551                         fl.oif = dev_out->ifindex;
2552                         goto make_route;
2553                 }
2554
2555                 if (!(oldflp->flags & FLOWI_FLAG_ANYSRC)) {
2556                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2557                         if (!__ip_dev_find(net, oldflp->fl4_src, false))
2558                                 goto out;
2559                 }
2560         }
2561
2562
2563         if (oldflp->oif) {
2564                 dev_out = dev_get_by_index_rcu(net, oldflp->oif);
2565                 err = -ENODEV;
2566                 if (dev_out == NULL)
2567                         goto out;
2568
2569                 /* RACE: Check return value of inet_select_addr instead. */
2570                 if (rcu_dereference(dev_out->ip_ptr) == NULL)
2571                         goto out;       /* Wrong error code */
2572
2573                 if (ipv4_is_local_multicast(oldflp->fl4_dst) ||
2574                     oldflp->fl4_dst == htonl(0xFFFFFFFF)) {
2575                         if (!fl.fl4_src)
2576                                 fl.fl4_src = inet_select_addr(dev_out, 0,
2577                                                               RT_SCOPE_LINK);
2578                         goto make_route;
2579                 }
2580                 if (!fl.fl4_src) {
2581                         if (ipv4_is_multicast(oldflp->fl4_dst))
2582                                 fl.fl4_src = inet_select_addr(dev_out, 0,
2583                                                               fl.fl4_scope);
2584                         else if (!oldflp->fl4_dst)
2585                                 fl.fl4_src = inet_select_addr(dev_out, 0,
2586                                                               RT_SCOPE_HOST);
2587                 }
2588         }
2589
2590         if (!fl.fl4_dst) {
2591                 fl.fl4_dst = fl.fl4_src;
2592                 if (!fl.fl4_dst)
2593                         fl.fl4_dst = fl.fl4_src = htonl(INADDR_LOOPBACK);
2594                 dev_out = net->loopback_dev;
2595                 fl.oif = net->loopback_dev->ifindex;
2596                 res.type = RTN_LOCAL;
2597                 flags |= RTCF_LOCAL;
2598                 goto make_route;
2599         }
2600
2601         if (fib_lookup(net, &fl, &res)) {
2602                 res.fi = NULL;
2603                 if (oldflp->oif) {
2604                         /* Apparently, routing tables are wrong. Assume,
2605                            that the destination is on link.
2606
2607                            WHY? DW.
2608                            Because we are allowed to send to iface
2609                            even if it has NO routes and NO assigned
2610                            addresses. When oif is specified, routing
2611                            tables are looked up with only one purpose:
2612                            to catch if destination is gatewayed, rather than
2613                            direct. Moreover, if MSG_DONTROUTE is set,
2614                            we send packet, ignoring both routing tables
2615                            and ifaddr state. --ANK
2616
2617
2618                            We could make it even if oif is unknown,
2619                            likely IPv6, but we do not.
2620                          */
2621
2622                         if (fl.fl4_src == 0)
2623                                 fl.fl4_src = inet_select_addr(dev_out, 0,
2624                                                               RT_SCOPE_LINK);
2625                         res.type = RTN_UNICAST;
2626                         goto make_route;
2627                 }
2628                 err = -ENETUNREACH;
2629                 goto out;
2630         }
2631
2632         if (res.type == RTN_LOCAL) {
2633                 if (!fl.fl4_src)
2634                         fl.fl4_src = fl.fl4_dst;
2635                 dev_out = net->loopback_dev;
2636                 fl.oif = dev_out->ifindex;
2637                 res.fi = NULL;
2638                 flags |= RTCF_LOCAL;
2639                 goto make_route;
2640         }
2641
2642 #ifdef CONFIG_IP_ROUTE_MULTIPATH
2643         if (res.fi->fib_nhs > 1 && fl.oif == 0)
2644                 fib_select_multipath(&fl, &res);
2645         else
2646 #endif
2647         if (!res.prefixlen && res.type == RTN_UNICAST && !fl.oif)
2648                 fib_select_default(net, &fl, &res);
2649
2650         if (!fl.fl4_src)
2651                 fl.fl4_src = FIB_RES_PREFSRC(res);
2652
2653         dev_out = FIB_RES_DEV(res);
2654         fl.oif = dev_out->ifindex;
2655
2656
2657 make_route:
2658         err = ip_mkroute_output(rp, &res, &fl, oldflp, dev_out, flags);
2659
2660 out:    return err;
2661 }
2662
2663 int __ip_route_output_key(struct net *net, struct rtable **rp,
2664                           const struct flowi *flp)
2665 {
2666         unsigned int hash;
2667         int res;
2668         struct rtable *rth;
2669
2670         if (!rt_caching(net))
2671                 goto slow_output;
2672
2673         hash = rt_hash(flp->fl4_dst, flp->fl4_src, flp->oif, rt_genid(net));
2674
2675         rcu_read_lock_bh();
2676         for (rth = rcu_dereference_bh(rt_hash_table[hash].chain); rth;
2677                 rth = rcu_dereference_bh(rth->dst.rt_next)) {
2678                 if (rth->fl.fl4_dst == flp->fl4_dst &&
2679                     rth->fl.fl4_src == flp->fl4_src &&
2680                     rth->fl.iif == 0 &&
2681                     rth->fl.oif == flp->oif &&
2682                     rth->fl.mark == flp->mark &&
2683                     !((rth->fl.fl4_tos ^ flp->fl4_tos) &
2684                             (IPTOS_RT_MASK | RTO_ONLINK)) &&
2685                     net_eq(dev_net(rth->dst.dev), net) &&
2686                     !rt_is_expired(rth)) {
2687                         dst_use(&rth->dst, jiffies);
2688                         RT_CACHE_STAT_INC(out_hit);
2689                         rcu_read_unlock_bh();
2690                         *rp = rth;
2691                         return 0;
2692                 }
2693                 RT_CACHE_STAT_INC(out_hlist_search);
2694         }
2695         rcu_read_unlock_bh();
2696
2697 slow_output:
2698         rcu_read_lock();
2699         res = ip_route_output_slow(net, rp, flp);
2700         rcu_read_unlock();
2701         return res;
2702 }
2703 EXPORT_SYMBOL_GPL(__ip_route_output_key);
2704
2705 static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
2706 {
2707         return NULL;
2708 }
2709
2710 static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, u32 mtu)
2711 {
2712 }
2713
2714 static struct dst_ops ipv4_dst_blackhole_ops = {
2715         .family                 =       AF_INET,
2716         .protocol               =       cpu_to_be16(ETH_P_IP),
2717         .destroy                =       ipv4_dst_destroy,
2718         .check                  =       ipv4_blackhole_dst_check,
2719         .update_pmtu            =       ipv4_rt_blackhole_update_pmtu,
2720         .entries                =       ATOMIC_INIT(0),
2721 };
2722
2723
2724 static int ipv4_dst_blackhole(struct net *net, struct rtable **rp, struct flowi *flp)
2725 {
2726         struct rtable *ort = *rp;
2727         struct rtable *rt = (struct rtable *)
2728                 dst_alloc(&ipv4_dst_blackhole_ops);
2729
2730         if (rt) {
2731                 struct dst_entry *new = &rt->dst;
2732
2733                 atomic_set(&new->__refcnt, 1);
2734                 new->__use = 1;
2735                 new->input = dst_discard;
2736                 new->output = dst_discard;
2737                 memcpy(new->metrics, ort->dst.metrics, RTAX_MAX*sizeof(u32));
2738
2739                 new->dev = ort->dst.dev;
2740                 if (new->dev)
2741                         dev_hold(new->dev);
2742
2743                 rt->fl = ort->fl;
2744
2745                 rt->idev = ort->idev;
2746                 if (rt->idev)
2747                         in_dev_hold(rt->idev);
2748                 rt->rt_genid = rt_genid(net);
2749                 rt->rt_flags = ort->rt_flags;
2750                 rt->rt_type = ort->rt_type;
2751                 rt->rt_dst = ort->rt_dst;
2752                 rt->rt_src = ort->rt_src;
2753                 rt->rt_iif = ort->rt_iif;
2754                 rt->rt_gateway = ort->rt_gateway;
2755                 rt->rt_spec_dst = ort->rt_spec_dst;
2756                 rt->peer = ort->peer;
2757                 if (rt->peer)
2758                         atomic_inc(&rt->peer->refcnt);
2759
2760                 dst_free(new);
2761         }
2762
2763         dst_release(&(*rp)->dst);
2764         *rp = rt;
2765         return rt ? 0 : -ENOMEM;
2766 }
2767
2768 int ip_route_output_flow(struct net *net, struct rtable **rp, struct flowi *flp,
2769                          struct sock *sk, int flags)
2770 {
2771         int err;
2772
2773         if ((err = __ip_route_output_key(net, rp, flp)) != 0)
2774                 return err;
2775
2776         if (flp->proto) {
2777                 if (!flp->fl4_src)
2778                         flp->fl4_src = (*rp)->rt_src;
2779                 if (!flp->fl4_dst)
2780                         flp->fl4_dst = (*rp)->rt_dst;
2781                 err = __xfrm_lookup(net, (struct dst_entry **)rp, flp, sk,
2782                                     flags ? XFRM_LOOKUP_WAIT : 0);
2783                 if (err == -EREMOTE)
2784                         err = ipv4_dst_blackhole(net, rp, flp);
2785
2786                 return err;
2787         }
2788
2789         return 0;
2790 }
2791 EXPORT_SYMBOL_GPL(ip_route_output_flow);
2792
2793 int ip_route_output_key(struct net *net, struct rtable **rp, struct flowi *flp)
2794 {
2795         return ip_route_output_flow(net, rp, flp, NULL, 0);
2796 }
2797 EXPORT_SYMBOL(ip_route_output_key);
2798
2799 static int rt_fill_info(struct net *net,
2800                         struct sk_buff *skb, u32 pid, u32 seq, int event,
2801                         int nowait, unsigned int flags)
2802 {
2803         struct rtable *rt = skb_rtable(skb);
2804         struct rtmsg *r;
2805         struct nlmsghdr *nlh;
2806         long expires;
2807         u32 id = 0, ts = 0, tsage = 0, error;
2808
2809         nlh = nlmsg_put(skb, pid, seq, event, sizeof(*r), flags);
2810         if (nlh == NULL)
2811                 return -EMSGSIZE;
2812
2813         r = nlmsg_data(nlh);
2814         r->rtm_family    = AF_INET;
2815         r->rtm_dst_len  = 32;
2816         r->rtm_src_len  = 0;
2817         r->rtm_tos      = rt->fl.fl4_tos;
2818         r->rtm_table    = RT_TABLE_MAIN;
2819         NLA_PUT_U32(skb, RTA_TABLE, RT_TABLE_MAIN);
2820         r->rtm_type     = rt->rt_type;
2821         r->rtm_scope    = RT_SCOPE_UNIVERSE;
2822         r->rtm_protocol = RTPROT_UNSPEC;
2823         r->rtm_flags    = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
2824         if (rt->rt_flags & RTCF_NOTIFY)
2825                 r->rtm_flags |= RTM_F_NOTIFY;
2826
2827         NLA_PUT_BE32(skb, RTA_DST, rt->rt_dst);
2828
2829         if (rt->fl.fl4_src) {
2830                 r->rtm_src_len = 32;
2831                 NLA_PUT_BE32(skb, RTA_SRC, rt->fl.fl4_src);
2832         }
2833         if (rt->dst.dev)
2834                 NLA_PUT_U32(skb, RTA_OIF, rt->dst.dev->ifindex);
2835 #ifdef CONFIG_NET_CLS_ROUTE
2836         if (rt->dst.tclassid)
2837                 NLA_PUT_U32(skb, RTA_FLOW, rt->dst.tclassid);
2838 #endif
2839         if (rt->fl.iif)
2840                 NLA_PUT_BE32(skb, RTA_PREFSRC, rt->rt_spec_dst);
2841         else if (rt->rt_src != rt->fl.fl4_src)
2842                 NLA_PUT_BE32(skb, RTA_PREFSRC, rt->rt_src);
2843
2844         if (rt->rt_dst != rt->rt_gateway)
2845                 NLA_PUT_BE32(skb, RTA_GATEWAY, rt->rt_gateway);
2846
2847         if (rtnetlink_put_metrics(skb, rt->dst.metrics) < 0)
2848                 goto nla_put_failure;
2849
2850         if (rt->fl.mark)
2851                 NLA_PUT_BE32(skb, RTA_MARK, rt->fl.mark);
2852
2853         error = rt->dst.error;
2854         expires = rt->dst.expires ? rt->dst.expires - jiffies : 0;
2855         if (rt->peer) {
2856                 inet_peer_refcheck(rt->peer);
2857                 id = atomic_read(&rt->peer->ip_id_count) & 0xffff;
2858                 if (rt->peer->tcp_ts_stamp) {
2859                         ts = rt->peer->tcp_ts;
2860                         tsage = get_seconds() - rt->peer->tcp_ts_stamp;
2861                 }
2862         }
2863
2864         if (rt->fl.iif) {
2865 #ifdef CONFIG_IP_MROUTE
2866                 __be32 dst = rt->rt_dst;
2867
2868                 if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
2869                     IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
2870                         int err = ipmr_get_route(net, skb, r, nowait);
2871                         if (err <= 0) {
2872                                 if (!nowait) {
2873                                         if (err == 0)
2874                                                 return 0;
2875                                         goto nla_put_failure;
2876                                 } else {
2877                                         if (err == -EMSGSIZE)
2878                                                 goto nla_put_failure;
2879                                         error = err;
2880                                 }
2881                         }
2882                 } else
2883 #endif
2884                         NLA_PUT_U32(skb, RTA_IIF, rt->fl.iif);
2885         }
2886
2887         if (rtnl_put_cacheinfo(skb, &rt->dst, id, ts, tsage,
2888                                expires, error) < 0)
2889                 goto nla_put_failure;
2890
2891         return nlmsg_end(skb, nlh);
2892
2893 nla_put_failure:
2894         nlmsg_cancel(skb, nlh);
2895         return -EMSGSIZE;
2896 }
2897
2898 static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr* nlh, void *arg)
2899 {
2900         struct net *net = sock_net(in_skb->sk);
2901         struct rtmsg *rtm;
2902         struct nlattr *tb[RTA_MAX+1];
2903         struct rtable *rt = NULL;
2904         __be32 dst = 0;
2905         __be32 src = 0;
2906         u32 iif;
2907         int err;
2908         int mark;
2909         struct sk_buff *skb;
2910
2911         err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy);
2912         if (err < 0)
2913                 goto errout;
2914
2915         rtm = nlmsg_data(nlh);
2916
2917         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2918         if (skb == NULL) {
2919                 err = -ENOBUFS;
2920                 goto errout;
2921         }
2922
2923         /* Reserve room for dummy headers, this skb can pass
2924            through good chunk of routing engine.
2925          */
2926         skb_reset_mac_header(skb);
2927         skb_reset_network_header(skb);
2928
2929         /* Bugfix: need to give ip_route_input enough of an IP header to not gag. */
2930         ip_hdr(skb)->protocol = IPPROTO_ICMP;
2931         skb_reserve(skb, MAX_HEADER + sizeof(struct iphdr));
2932
2933         src = tb[RTA_SRC] ? nla_get_be32(tb[RTA_SRC]) : 0;
2934         dst = tb[RTA_DST] ? nla_get_be32(tb[RTA_DST]) : 0;
2935         iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
2936         mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
2937
2938         if (iif) {
2939                 struct net_device *dev;
2940
2941                 dev = __dev_get_by_index(net, iif);
2942                 if (dev == NULL) {
2943                         err = -ENODEV;
2944                         goto errout_free;
2945                 }
2946
2947                 skb->protocol   = htons(ETH_P_IP);
2948                 skb->dev        = dev;
2949                 skb->mark       = mark;
2950                 local_bh_disable();
2951                 err = ip_route_input(skb, dst, src, rtm->rtm_tos, dev);
2952                 local_bh_enable();
2953
2954                 rt = skb_rtable(skb);
2955                 if (err == 0 && rt->dst.error)
2956                         err = -rt->dst.error;
2957         } else {
2958                 struct flowi fl = {
2959                         .nl_u = {
2960                                 .ip4_u = {
2961                                         .daddr = dst,
2962                                         .saddr = src,
2963                                         .tos = rtm->rtm_tos,
2964                                 },
2965                         },
2966                         .oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0,
2967                         .mark = mark,
2968                 };
2969                 err = ip_route_output_key(net, &rt, &fl);
2970         }
2971
2972         if (err)
2973                 goto errout_free;
2974
2975         skb_dst_set(skb, &rt->dst);
2976         if (rtm->rtm_flags & RTM_F_NOTIFY)
2977                 rt->rt_flags |= RTCF_NOTIFY;
2978
2979         err = rt_fill_info(net, skb, NETLINK_CB(in_skb).pid, nlh->nlmsg_seq,
2980                            RTM_NEWROUTE, 0, 0);
2981         if (err <= 0)
2982                 goto errout_free;
2983
2984         err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).pid);
2985 errout:
2986         return err;
2987
2988 errout_free:
2989         kfree_skb(skb);
2990         goto errout;
2991 }
2992
2993 int ip_rt_dump(struct sk_buff *skb,  struct netlink_callback *cb)
2994 {
2995         struct rtable *rt;
2996         int h, s_h;
2997         int idx, s_idx;
2998         struct net *net;
2999
3000         net = sock_net(skb->sk);
3001
3002         s_h = cb->args[0];
3003         if (s_h < 0)
3004                 s_h = 0;
3005         s_idx = idx = cb->args[1];
3006         for (h = s_h; h <= rt_hash_mask; h++, s_idx = 0) {
3007                 if (!rt_hash_table[h].chain)
3008                         continue;
3009                 rcu_read_lock_bh();
3010                 for (rt = rcu_dereference_bh(rt_hash_table[h].chain), idx = 0; rt;
3011                      rt = rcu_dereference_bh(rt->dst.rt_next), idx++) {
3012                         if (!net_eq(dev_net(rt->dst.dev), net) || idx < s_idx)
3013                                 continue;
3014                         if (rt_is_expired(rt))
3015                                 continue;
3016                         skb_dst_set_noref(skb, &rt->dst);
3017                         if (rt_fill_info(net, skb, NETLINK_CB(cb->skb).pid,
3018                                          cb->nlh->nlmsg_seq, RTM_NEWROUTE,
3019                                          1, NLM_F_MULTI) <= 0) {
3020                                 skb_dst_drop(skb);
3021                                 rcu_read_unlock_bh();
3022                                 goto done;
3023                         }
3024                         skb_dst_drop(skb);
3025                 }
3026                 rcu_read_unlock_bh();
3027         }
3028
3029 done:
3030         cb->args[0] = h;
3031         cb->args[1] = idx;
3032         return skb->len;
3033 }
3034
3035 void ip_rt_multicast_event(struct in_device *in_dev)
3036 {
3037         rt_cache_flush(dev_net(in_dev->dev), 0);
3038 }
3039
3040 #ifdef CONFIG_SYSCTL
3041 static int ipv4_sysctl_rtcache_flush(ctl_table *__ctl, int write,
3042                                         void __user *buffer,
3043                                         size_t *lenp, loff_t *ppos)
3044 {
3045         if (write) {
3046                 int flush_delay;
3047                 ctl_table ctl;
3048                 struct net *net;
3049
3050                 memcpy(&ctl, __ctl, sizeof(ctl));
3051                 ctl.data = &flush_delay;
3052                 proc_dointvec(&ctl, write, buffer, lenp, ppos);
3053
3054                 net = (struct net *)__ctl->extra1;
3055                 rt_cache_flush(net, flush_delay);
3056                 return 0;
3057         }
3058
3059         return -EINVAL;
3060 }
3061
3062 static ctl_table ipv4_route_table[] = {
3063         {
3064                 .procname       = "gc_thresh",
3065                 .data           = &ipv4_dst_ops.gc_thresh,
3066                 .maxlen         = sizeof(int),
3067                 .mode           = 0644,
3068                 .proc_handler   = proc_dointvec,
3069         },
3070         {
3071                 .procname       = "max_size",
3072                 .data           = &ip_rt_max_size,
3073                 .maxlen         = sizeof(int),
3074                 .mode           = 0644,
3075                 .proc_handler   = proc_dointvec,
3076         },
3077         {
3078                 /*  Deprecated. Use gc_min_interval_ms */
3079
3080                 .procname       = "gc_min_interval",
3081                 .data           = &ip_rt_gc_min_interval,
3082                 .maxlen         = sizeof(int),
3083                 .mode           = 0644,
3084                 .proc_handler   = proc_dointvec_jiffies,
3085         },
3086         {
3087                 .procname       = "gc_min_interval_ms",
3088                 .data           = &ip_rt_gc_min_interval,
3089                 .maxlen         = sizeof(int),
3090                 .mode           = 0644,
3091                 .proc_handler   = proc_dointvec_ms_jiffies,
3092         },
3093         {
3094                 .procname       = "gc_timeout",
3095                 .data           = &ip_rt_gc_timeout,
3096                 .maxlen         = sizeof(int),
3097                 .mode           = 0644,
3098                 .proc_handler   = proc_dointvec_jiffies,
3099         },
3100         {
3101                 .procname       = "gc_interval",
3102                 .data           = &ip_rt_gc_interval,
3103                 .maxlen         = sizeof(int),
3104                 .mode           = 0644,
3105                 .proc_handler   = proc_dointvec_jiffies,
3106         },
3107         {
3108                 .procname       = "redirect_load",
3109                 .data           = &ip_rt_redirect_load,
3110                 .maxlen         = sizeof(int),
3111                 .mode           = 0644,
3112                 .proc_handler   = proc_dointvec,
3113         },
3114         {
3115                 .procname       = "redirect_number",
3116                 .data           = &ip_rt_redirect_number,
3117                 .maxlen         = sizeof(int),
3118                 .mode           = 0644,
3119                 .proc_handler   = proc_dointvec,
3120         },
3121         {
3122                 .procname       = "redirect_silence",
3123                 .data           = &ip_rt_redirect_silence,
3124                 .maxlen         = sizeof(int),
3125                 .mode           = 0644,
3126                 .proc_handler   = proc_dointvec,
3127         },
3128         {
3129                 .procname       = "error_cost",
3130                 .data           = &ip_rt_error_cost,
3131                 .maxlen         = sizeof(int),
3132                 .mode           = 0644,
3133                 .proc_handler   = proc_dointvec,
3134         },
3135         {
3136                 .procname       = "error_burst",
3137                 .data           = &ip_rt_error_burst,
3138                 .maxlen         = sizeof(int),
3139                 .mode           = 0644,
3140                 .proc_handler   = proc_dointvec,
3141         },
3142         {
3143                 .procname       = "gc_elasticity",
3144                 .data           = &ip_rt_gc_elasticity,
3145                 .maxlen         = sizeof(int),
3146                 .mode           = 0644,
3147                 .proc_handler   = proc_dointvec,
3148         },
3149         {
3150                 .procname       = "mtu_expires",
3151                 .data           = &ip_rt_mtu_expires,
3152                 .maxlen         = sizeof(int),
3153                 .mode           = 0644,
3154                 .proc_handler   = proc_dointvec_jiffies,
3155         },
3156         {
3157                 .procname       = "min_pmtu",
3158                 .data           = &ip_rt_min_pmtu,
3159                 .maxlen         = sizeof(int),
3160                 .mode           = 0644,
3161                 .proc_handler   = proc_dointvec,
3162         },
3163         {
3164                 .procname       = "min_adv_mss",
3165                 .data           = &ip_rt_min_advmss,
3166                 .maxlen         = sizeof(int),
3167                 .mode           = 0644,
3168                 .proc_handler   = proc_dointvec,
3169         },
3170         { }
3171 };
3172
3173 static struct ctl_table empty[1];
3174
3175 static struct ctl_table ipv4_skeleton[] =
3176 {
3177         { .procname = "route",
3178           .mode = 0555, .child = ipv4_route_table},
3179         { .procname = "neigh",
3180           .mode = 0555, .child = empty},
3181         { }
3182 };
3183
3184 static __net_initdata struct ctl_path ipv4_path[] = {
3185         { .procname = "net", },
3186         { .procname = "ipv4", },
3187         { },
3188 };
3189
3190 static struct ctl_table ipv4_route_flush_table[] = {
3191         {
3192                 .procname       = "flush",
3193                 .maxlen         = sizeof(int),
3194                 .mode           = 0200,
3195                 .proc_handler   = ipv4_sysctl_rtcache_flush,
3196         },
3197         { },
3198 };
3199
3200 static __net_initdata struct ctl_path ipv4_route_path[] = {
3201         { .procname = "net", },
3202         { .procname = "ipv4", },
3203         { .procname = "route", },
3204         { },
3205 };
3206
3207 static __net_init int sysctl_route_net_init(struct net *net)
3208 {
3209         struct ctl_table *tbl;
3210
3211         tbl = ipv4_route_flush_table;
3212         if (!net_eq(net, &init_net)) {
3213                 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
3214                 if (tbl == NULL)
3215                         goto err_dup;
3216         }
3217         tbl[0].extra1 = net;
3218
3219         net->ipv4.route_hdr =
3220                 register_net_sysctl_table(net, ipv4_route_path, tbl);
3221         if (net->ipv4.route_hdr == NULL)
3222                 goto err_reg;
3223         return 0;
3224
3225 err_reg:
3226         if (tbl != ipv4_route_flush_table)
3227                 kfree(tbl);
3228 err_dup:
3229         return -ENOMEM;
3230 }
3231
3232 static __net_exit void sysctl_route_net_exit(struct net *net)
3233 {
3234         struct ctl_table *tbl;
3235
3236         tbl = net->ipv4.route_hdr->ctl_table_arg;
3237         unregister_net_sysctl_table(net->ipv4.route_hdr);
3238         BUG_ON(tbl == ipv4_route_flush_table);
3239         kfree(tbl);
3240 }
3241
3242 static __net_initdata struct pernet_operations sysctl_route_ops = {
3243         .init = sysctl_route_net_init,
3244         .exit = sysctl_route_net_exit,
3245 };
3246 #endif
3247
3248 static __net_init int rt_genid_init(struct net *net)
3249 {
3250         get_random_bytes(&net->ipv4.rt_genid,
3251                          sizeof(net->ipv4.rt_genid));
3252         return 0;
3253 }
3254
3255 static __net_initdata struct pernet_operations rt_genid_ops = {
3256         .init = rt_genid_init,
3257 };
3258
3259
3260 #ifdef CONFIG_NET_CLS_ROUTE
3261 struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
3262 #endif /* CONFIG_NET_CLS_ROUTE */
3263
3264 static __initdata unsigned long rhash_entries;
3265 static int __init set_rhash_entries(char *str)
3266 {
3267         if (!str)
3268                 return 0;
3269         rhash_entries = simple_strtoul(str, &str, 0);
3270         return 1;
3271 }
3272 __setup("rhash_entries=", set_rhash_entries);
3273
3274 int __init ip_rt_init(void)
3275 {
3276         int rc = 0;
3277
3278 #ifdef CONFIG_NET_CLS_ROUTE
3279         ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
3280         if (!ip_rt_acct)
3281                 panic("IP: failed to allocate ip_rt_acct\n");
3282 #endif
3283
3284         ipv4_dst_ops.kmem_cachep =
3285                 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
3286                                   SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
3287
3288         ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;
3289
3290         rt_hash_table = (struct rt_hash_bucket *)
3291                 alloc_large_system_hash("IP route cache",
3292                                         sizeof(struct rt_hash_bucket),
3293                                         rhash_entries,
3294                                         (totalram_pages >= 128 * 1024) ?
3295                                         15 : 17,
3296                                         0,
3297                                         &rt_hash_log,
3298                                         &rt_hash_mask,
3299                                         rhash_entries ? 0 : 512 * 1024);
3300         memset(rt_hash_table, 0, (rt_hash_mask + 1) * sizeof(struct rt_hash_bucket));
3301         rt_hash_lock_init();
3302
3303         ipv4_dst_ops.gc_thresh = (rt_hash_mask + 1);
3304         ip_rt_max_size = (rt_hash_mask + 1) * 16;
3305
3306         devinet_init();
3307         ip_fib_init();
3308
3309         /* All the timers, started at system startup tend
3310            to synchronize. Perturb it a bit.
3311          */
3312         INIT_DELAYED_WORK_DEFERRABLE(&expires_work, rt_worker_func);
3313         expires_ljiffies = jiffies;
3314         schedule_delayed_work(&expires_work,
3315                 net_random() % ip_rt_gc_interval + ip_rt_gc_interval);
3316
3317         if (ip_rt_proc_init())
3318                 printk(KERN_ERR "Unable to create route proc files\n");
3319 #ifdef CONFIG_XFRM
3320         xfrm_init();
3321         xfrm4_init(ip_rt_max_size);
3322 #endif
3323         rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL);
3324
3325 #ifdef CONFIG_SYSCTL
3326         register_pernet_subsys(&sysctl_route_ops);
3327 #endif
3328         register_pernet_subsys(&rt_genid_ops);
3329         return rc;
3330 }
3331
3332 #ifdef CONFIG_SYSCTL
3333 /*
3334  * We really need to sanitize the damn ipv4 init order, then all
3335  * this nonsense will go away.
3336  */
3337 void __init ip_static_sysctl_init(void)
3338 {
3339         register_sysctl_paths(ipv4_path, ipv4_skeleton);
3340 }
3341 #endif