mm: avoid false sharing of mm_counter

[net-next-2.6.git] / include / linux / mm_types.h

diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 84d020bed0830eecb4ffd3e678d2c64f7ff4ddc1..21861239ab0cb15f0eb82428aafa04049b255f12 100644 (file)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -24,12 +24,6 @@ struct address_space;
 
 #define USE_SPLIT_PTLOCKS      (NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS)
 
-#if USE_SPLIT_PTLOCKS
-typedef atomic_long_t mm_counter_t;
-#else  /* !USE_SPLIT_PTLOCKS */
-typedef unsigned long mm_counter_t;
-#endif /* !USE_SPLIT_PTLOCKS */
-
 /*
  * Each physical page in the system has a struct page associated with
  * it to keep track of whatever it is we are using the page for at the
@@ -122,7 +116,7 @@ struct vm_region {
        unsigned long   vm_pgoff;       /* the offset in vm_file corresponding to vm_start */
        struct file     *vm_file;       /* the backing file or NULL */
 
-       atomic_t        vm_usage;       /* region usage count */
+       int             vm_usage;       /* region usage count (access under nommu_region_sem) */
        bool            vm_icache_flushed : 1; /* true if the icache has been flushed for
                                                * this region */
 };
@@ -201,14 +195,38 @@ struct core_state {
        struct completion startup;
 };
 
+enum {
+       MM_FILEPAGES,
+       MM_ANONPAGES,
+       NR_MM_COUNTERS
+};
+
+#if USE_SPLIT_PTLOCKS
+#define SPLIT_RSS_COUNTING
+struct mm_rss_stat {
+       atomic_long_t count[NR_MM_COUNTERS];
+};
+/* per-thread cached information, */
+struct task_rss_stat {
+       int events;     /* for synchronization threshold */
+       int count[NR_MM_COUNTERS];
+};
+#else  /* !USE_SPLIT_PTLOCKS */
+struct mm_rss_stat {
+       unsigned long count[NR_MM_COUNTERS];
+};
+#endif /* !USE_SPLIT_PTLOCKS */
+
 struct mm_struct {
        struct vm_area_struct * mmap;           /* list of VMAs */
        struct rb_root mm_rb;
        struct vm_area_struct * mmap_cache;     /* last find_vma result */
+#ifdef CONFIG_MMU
        unsigned long (*get_unmapped_area) (struct file *filp,
                                unsigned long addr, unsigned long len,
                                unsigned long pgoff, unsigned long flags);
        void (*unmap_area) (struct mm_struct *mm, unsigned long addr);
+#endif
        unsigned long mmap_base;                /* base of mmap area */
        unsigned long task_size;                /* size of task vm space */
        unsigned long cached_hole_size;         /* if non-zero, the largest hole below free_area_cache */
@@ -225,11 +243,6 @@ struct mm_struct {
                                                 * by mmlist_lock
                                                 */
 
-       /* Special counters, in some configurations protected by the
-        * page_table_lock, in other configurations by being atomic.
-        */
-       mm_counter_t _file_rss;
-       mm_counter_t _anon_rss;
 
        unsigned long hiwater_rss;      /* High-watermark of RSS usage */
        unsigned long hiwater_vm;       /* High-water virtual memory usage */
@@ -242,6 +255,12 @@ struct mm_struct {
 
        unsigned long saved_auxv[AT_VECTOR_SIZE]; /* for /proc/PID/auxv */
 
+       /*
+        * Special counters, in some configurations protected by the
+        * page_table_lock, in other configurations by being atomic.
+        */
+       struct mm_rss_stat rss_stat;
+
        struct linux_binfmt *binfmt;
 
        cpumask_t cpu_vm_mask;