static inline struct net *nf_ct_net(const struct nf_conn *ct)
{
- #ifdef CONFIG_NET_NS
- return ct->ct_net;
- #else
- return &init_net;
- #endif
+ return read_pnet(&ct->ct_net);
}
/* Alter reply tuple (maybe alter helper). */
u32 seq);
/* Fake conntrack entry for untracked connections */
-extern struct nf_conn nf_conntrack_untracked;
+DECLARE_PER_CPU(struct nf_conn, nf_conntrack_untracked);
+static inline struct nf_conn *nf_ct_untracked_get(void)
+{
+ return &__raw_get_cpu_var(nf_conntrack_untracked);
+}
+extern void nf_ct_untracked_status_or(unsigned long bits);
/* Iterate over all conntracks: if iter returns true, it's deleted. */
extern void
return test_bit(IPS_DYING_BIT, &ct->status);
}
-static inline int nf_ct_is_untracked(const struct sk_buff *skb)
+static inline int nf_ct_is_untracked(const struct nf_conn *ct)
{
- return (skb->nfct == &nf_conntrack_untracked.ct_general);
+ return test_bit(IPS_UNTRACKED_BIT, &ct->status);
}
extern int nf_conntrack_set_hashsize(const char *val, struct kernel_param *kp);
#define _XT_RATEEST_H
struct xt_rateest {
+ /* keep lock and bstats on same cache line to speedup xt_rateest_tg() */
+ struct gnet_stats_basic_packed bstats;
+ spinlock_t lock;
+ /* keep rstats and lock on same cache line to speedup xt_rateest_mt() */
+ struct gnet_stats_rate_est rstats;
+
+ /* following fields not accessed in hot path */
struct hlist_node list;
char name[IFNAMSIZ];
unsigned int refcnt;
- spinlock_t lock;
struct gnet_estimator params;
- struct gnet_stats_rate_est rstats;
- struct gnet_stats_basic_packed bstats;
+ struct rcu_head rcu;
};
extern struct xt_rateest *xt_rateest_lookup(const char *name);
{
struct rtable *rt = &br->fake_rtable;
- atomic_set(&rt->u.dst.__refcnt, 1);
- rt->u.dst.dev = br->dev;
- rt->u.dst.path = &rt->u.dst;
- rt->u.dst.metrics[RTAX_MTU - 1] = 1500;
- rt->u.dst.flags = DST_NOXFRM;
- rt->u.dst.ops = &fake_dst_ops;
+ atomic_set(&rt->dst.__refcnt, 1);
+ rt->dst.dev = br->dev;
+ rt->dst.path = &rt->dst;
+ rt->dst.metrics[RTAX_MTU - 1] = 1500;
+ rt->dst.flags = DST_NOXFRM;
+ rt->dst.ops = &fake_dst_ops;
}
static inline struct rtable *bridge_parent_rtable(const struct net_device *dev)
kfree_skb(skb);
return 0;
}
- skb_dst_set_noref(skb, &rt->u.dst);
- dst_hold(&rt->dst);
- skb_dst_set(skb, &rt->dst);
++ skb_dst_set_noref(skb, &rt->dst);
skb->dev = nf_bridge->physindev;
nf_bridge_update_protocol(skb);
kfree_skb(skb);
return 0;
}
- skb_dst_set_noref(skb, &rt->u.dst);
- dst_hold(&rt->dst);
- skb_dst_set(skb, &rt->dst);
++ skb_dst_set_noref(skb, &rt->dst);
}
skb->dev = nf_bridge->physindev;
/* Drop old route. */
skb_dst_drop(skb);
- skb_dst_set(skb, &rt->u.dst);
+ skb_dst_set(skb, &rt->dst);
} else {
/* non-local src, find valid iif to satisfy
* rp-filter when calling ip_route_input. */
orefdst = skb->_skb_refdst;
if (ip_route_input(skb, iph->daddr, iph->saddr,
- RT_TOS(iph->tos), rt->u.dst.dev) != 0) {
- dst_release(&rt->u.dst);
+ RT_TOS(iph->tos), rt->dst.dev) != 0) {
+ dst_release(&rt->dst);
return -1;
}
- dst_release(&rt->u.dst);
+ dst_release(&rt->dst);
refdst_drop(orefdst);
}
skb->csum = csum_tcpudp_nofold(iph->saddr, iph->daddr, protocol,
skb->len - dataoff, 0);
skb->ip_summed = CHECKSUM_NONE;
- csum = __skb_checksum_complete_head(skb, dataoff + len);
- if (!csum)
- skb->ip_summed = CHECKSUM_UNNECESSARY;
+ return __skb_checksum_complete_head(skb, dataoff + len);
}
return csum;
}
cpu = smp_processor_id();
table_base = private->entries[cpu];
jumpstack = (struct ipt_entry **)private->jumpstack[cpu];
- stackptr = &private->stackptr[cpu];
+ stackptr = per_cpu_ptr(private->stackptr, cpu);
origptr = *stackptr;
e = get_entry(table_base, private->hook_entry[hook]);
(other than comefrom, which userspace doesn't care
about). */
countersize = sizeof(struct xt_counters) * private->number;
- counters = vmalloc_node(countersize, numa_node_id());
+ counters = vmalloc(countersize);
if (counters == NULL)
return ERR_PTR(-ENOMEM);
if (len != size + num_counters * sizeof(struct xt_counters))
return -EINVAL;
- paddc = vmalloc_node(len - size, numa_node_id());
+ paddc = vmalloc(len - size);
if (!paddc)
return -ENOMEM;
cpu = smp_processor_id();
table_base = private->entries[cpu];
jumpstack = (struct ip6t_entry **)private->jumpstack[cpu];
- stackptr = &private->stackptr[cpu];
+ stackptr = per_cpu_ptr(private->stackptr, cpu);
origptr = *stackptr;
e = get_entry(table_base, private->hook_entry[hook]);
(other than comefrom, which userspace doesn't care
about). */
countersize = sizeof(struct xt_counters) * private->number;
- counters = vmalloc_node(countersize, numa_node_id());
+ counters = vmalloc(countersize);
if (counters == NULL)
return ERR_PTR(-ENOMEM);
struct ip6t_entry *iter;
ret = 0;
- counters = vmalloc_node(num_counters * sizeof(struct xt_counters),
- numa_node_id());
+ counters = vmalloc(num_counters * sizeof(struct xt_counters));
if (!counters) {
ret = -ENOMEM;
goto out;
if (len != size + num_counters * sizeof(struct xt_counters))
return -EINVAL;
- paddc = vmalloc_node(len - size, numa_node_id());
+ paddc = vmalloc(len - size);
if (!paddc)
return -ENOMEM;
unsigned int nf_conntrack_max __read_mostly;
EXPORT_SYMBOL_GPL(nf_conntrack_max);
-struct nf_conn nf_conntrack_untracked __read_mostly;
-EXPORT_SYMBOL_GPL(nf_conntrack_untracked);
+DEFINE_PER_CPU(struct nf_conn, nf_conntrack_untracked);
+EXPORT_PER_CPU_SYMBOL(nf_conntrack_untracked);
static int nf_conntrack_hash_rnd_initted;
static unsigned int nf_conntrack_hash_rnd;
ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev = NULL;
/* Don't set timer yet: wait for confirmation */
setup_timer(&ct->timeout, death_by_timeout, (unsigned long)ct);
- #ifdef CONFIG_NET_NS
- ct->ct_net = net;
- #endif
+ write_pnet(&ct->ct_net, net);
#ifdef CONFIG_NF_CONNTRACK_ZONES
if (zone) {
struct nf_conntrack_zone *nf_ct_zone;
spin_unlock_bh(&nf_conntrack_lock);
}
+static int untrack_refs(void)
+{
+ int cnt = 0, cpu;
+
+ for_each_possible_cpu(cpu) {
+ struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);
+
+ cnt += atomic_read(&ct->ct_general.use) - 1;
+ }
+ return cnt;
+}
+
static void nf_conntrack_cleanup_init_net(void)
{
- /* wait until all references to nf_conntrack_untracked are dropped */
- while (atomic_read(&nf_conntrack_untracked.ct_general.use) > 1)
+ while (untrack_refs() > 0)
schedule();
nf_conntrack_helper_fini();
module_param_call(hashsize, nf_conntrack_set_hashsize, param_get_uint,
&nf_conntrack_htable_size, 0600);
+void nf_ct_untracked_status_or(unsigned long bits)
+{
+ int cpu;
+
+ for_each_possible_cpu(cpu)
+ per_cpu(nf_conntrack_untracked, cpu).status |= bits;
+}
+EXPORT_SYMBOL_GPL(nf_ct_untracked_status_or);
+
static int nf_conntrack_init_init_net(void)
{
int max_factor = 8;
- int ret;
+ int ret, cpu;
/* Idea from tcp.c: use 1/16384 of memory. On i386: 32MB
* machine has 512 buckets. >= 1GB machines have 16384 buckets. */
goto err_extend;
#endif
/* Set up fake conntrack: to never be deleted, not in any hashes */
- write_pnet(&nf_conntrack_untracked.ct_net, &init_net);
- atomic_set(&nf_conntrack_untracked.ct_general.use, 1);
+ for_each_possible_cpu(cpu) {
+ struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);
-
+ write_pnet(&ct->ct_net, &init_net);
+ atomic_set(&ct->ct_general.use, 1);
+ }
/* - and look it like as a confirmed connection */
- set_bit(IPS_CONFIRMED_BIT, &nf_conntrack_untracked.status);
-
+ nf_ct_untracked_status_or(IPS_CONFIRMED | IPS_UNTRACKED);
return 0;
#ifdef CONFIG_NF_CONNTRACK_ZONES
return false;
skb_dst_drop(skb);
- skb_dst_set(skb, &rt->u.dst);
- skb->dev = rt->u.dst.dev;
+ skb_dst_set(skb, &rt->dst);
+ skb->dev = rt->dst.dev;
skb->protocol = htons(ETH_P_IP);
return true;
}
#ifdef WITH_CONNTRACK
/* Avoid counting cloned packets towards the original connection. */
nf_conntrack_put(skb->nfct);
- skb->nfct = &nf_conntrack_untracked.ct_general;
+ skb->nfct = &nf_ct_untracked_get()->ct_general;
skb->nfctinfo = IP_CT_NEW;
nf_conntrack_get(skb->nfct);
#endif
#ifdef WITH_CONNTRACK
nf_conntrack_put(skb->nfct);
- skb->nfct = &nf_conntrack_untracked.ct_general;
+ skb->nfct = &nf_ct_untracked_get()->ct_general;
skb->nfctinfo = IP_CT_NEW;
nf_conntrack_get(skb->nfct);
#endif
git://bbs.cooldavid.org / net-next-2.6.git / commitdiff