mm: avoid false sharing of mm_counter

[net-next-2.6.git] / include / linux / mm_types.h

diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 36f96271306c728f17ebfc23f538acd092ed2dac..21861239ab0cb15f0eb82428aafa04049b255f12 100644 (file)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -24,12 +24,6 @@ struct address_space;
 
 #define USE_SPLIT_PTLOCKS      (NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS)
 
-#if USE_SPLIT_PTLOCKS
-typedef atomic_long_t mm_counter_t;
-#else  /* !USE_SPLIT_PTLOCKS */
-typedef unsigned long mm_counter_t;
-#endif /* !USE_SPLIT_PTLOCKS */
-
 /*
  * Each physical page in the system has a struct page associated with
  * it to keep track of whatever it is we are using the page for at the
@@ -201,6 +195,28 @@ struct core_state {
        struct completion startup;
 };
 
+enum {
+       MM_FILEPAGES,
+       MM_ANONPAGES,
+       NR_MM_COUNTERS
+};
+
+#if USE_SPLIT_PTLOCKS
+#define SPLIT_RSS_COUNTING
+struct mm_rss_stat {
+       atomic_long_t count[NR_MM_COUNTERS];
+};
+/* per-thread cached information, */
+struct task_rss_stat {
+       int events;     /* for synchronization threshold */
+       int count[NR_MM_COUNTERS];
+};
+#else  /* !USE_SPLIT_PTLOCKS */
+struct mm_rss_stat {
+       unsigned long count[NR_MM_COUNTERS];
+};
+#endif /* !USE_SPLIT_PTLOCKS */
+
 struct mm_struct {
        struct vm_area_struct * mmap;           /* list of VMAs */
        struct rb_root mm_rb;
@@ -227,11 +243,6 @@ struct mm_struct {
                                                 * by mmlist_lock
                                                 */
 
-       /* Special counters, in some configurations protected by the
-        * page_table_lock, in other configurations by being atomic.
-        */
-       mm_counter_t _file_rss;
-       mm_counter_t _anon_rss;
 
        unsigned long hiwater_rss;      /* High-watermark of RSS usage */
        unsigned long hiwater_vm;       /* High-water virtual memory usage */
@@ -244,6 +255,12 @@ struct mm_struct {
 
        unsigned long saved_auxv[AT_VECTOR_SIZE]; /* for /proc/PID/auxv */
 
+       /*
+        * Special counters, in some configurations protected by the
+        * page_table_lock, in other configurations by being atomic.
+        */
+       struct mm_rss_stat rss_stat;
+
        struct linux_binfmt *binfmt;
 
        cpumask_t cpu_vm_mask;