net: Make RFS socket operations not be inet specific.

[net-next-2.6.git] / include / linux / netdevice.h

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 470f7c951afb481a43966936a81ce23fb559166b..c04ca246395db47d94dd1304c595e9ab031202e6 100644 (file)
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -530,14 +530,73 @@ struct rps_map {
 };
 #define RPS_MAP_SIZE(_num) (sizeof(struct rps_map) + (_num * sizeof(u16)))
 
+/*
+ * The rps_dev_flow structure contains the mapping of a flow to a CPU and the
+ * tail pointer for that CPU's input queue at the time of last enqueue.
+ */
+struct rps_dev_flow {
+       u16 cpu;
+       u16 fill;
+       unsigned int last_qtail;
+};
+
+/*
+ * The rps_dev_flow_table structure contains a table of flow mappings.
+ */
+struct rps_dev_flow_table {
+       unsigned int mask;
+       struct rcu_head rcu;
+       struct work_struct free_work;
+       struct rps_dev_flow flows[0];
+};
+#define RPS_DEV_FLOW_TABLE_SIZE(_num) (sizeof(struct rps_dev_flow_table) + \
+    (_num * sizeof(struct rps_dev_flow)))
+
+/*
+ * The rps_sock_flow_table contains mappings of flows to the last CPU
+ * on which they were processed by the application (set in recvmsg).
+ */
+struct rps_sock_flow_table {
+       unsigned int mask;
+       u16 ents[0];
+};
+#define        RPS_SOCK_FLOW_TABLE_SIZE(_num) (sizeof(struct rps_sock_flow_table) + \
+    (_num * sizeof(u16)))
+
+#define RPS_NO_CPU 0xffff
+
+static inline void rps_record_sock_flow(struct rps_sock_flow_table *table,
+                                       u32 hash)
+{
+       if (table && hash) {
+               unsigned int cpu, index = hash & table->mask;
+
+               /* We only give a hint, preemption can change cpu under us */
+               cpu = raw_smp_processor_id();
+
+               if (table->ents[index] != cpu)
+                       table->ents[index] = cpu;
+       }
+}
+
+static inline void rps_reset_sock_flow(struct rps_sock_flow_table *table,
+                                      u32 hash)
+{
+       if (table && hash)
+               table->ents[hash & table->mask] = RPS_NO_CPU;
+}
+
+extern struct rps_sock_flow_table *rps_sock_flow_table;
+
 /* This structure contains an instance of an RX queue. */
 struct netdev_rx_queue {
        struct rps_map *rps_map;
+       struct rps_dev_flow_table *rps_flow_table;
        struct kobject kobj;
        struct netdev_rx_queue *first;
        atomic_t count;
 } ____cacheline_aligned_in_smp;
-#endif
+#endif /* CONFIG_RPS */
 
 /*
  * This structure defines the management hooks for network devices.
@@ -1322,22 +1381,34 @@ static inline int unregister_gifconf(unsigned int family)
 }
 
 /*
- * Incoming packets are placed on per-cpu queues so that
- * no locking is needed.
+ * Incoming packets are placed on per-cpu queues
  */
 struct softnet_data {
        struct Qdisc            *output_queue;
+       struct Qdisc            **output_queue_tailp;
        struct list_head        poll_list;
        struct sk_buff          *completion_queue;
 
+#ifdef CONFIG_RPS
+       struct softnet_data     *rps_ipi_list;
+
        /* Elements below can be accessed between CPUs for RPS */
-#ifdef CONFIG_SMP
        struct call_single_data csd ____cacheline_aligned_in_smp;
+       struct softnet_data     *rps_ipi_next;
+       unsigned int            cpu;
+       unsigned int            input_queue_head;
 #endif
        struct sk_buff_head     input_pkt_queue;
        struct napi_struct      backlog;
 };
 
+static inline void input_queue_head_incr(struct softnet_data *sd)
+{
+#ifdef CONFIG_RPS
+       sd->input_queue_head++;
+#endif
+}
+
 DECLARE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
 
 #define HAVE_NETIF_QUEUE