block: cfq: make the io contect sharing lockless

[net-next-2.6.git] / block / ll_rw_blk.c

diff --git a/block/ll_rw_blk.c b/block/ll_rw_blk.c
index d4550ecae4431fd82610c1c8ca96255a7a94701f..b901db63f6ae082f1df23e3227c9ddd2e56d61d1 100644 (file)
--- a/block/ll_rw_blk.c
+++ b/block/ll_rw_blk.c
@@ -3853,6 +3853,21 @@ int __init blk_dev_init(void)
        return 0;
 }
 
+static void cfq_dtor(struct io_context *ioc)
+{
+       struct cfq_io_context *cic[1];
+       int r;
+
+       /*
+        * We don't have a specific key to lookup with, so use the gang
+        * lookup to just retrieve the first item stored. The cfq exit
+        * function will iterate the full tree, so any member will do.
+        */
+       r = radix_tree_gang_lookup(&ioc->radix_root, (void **) cic, 0, 1);
+       if (r > 0)
+               cic[0]->dtor(ioc);
+}
+
 /*
  * IO Context helper functions. put_io_context() returns 1 if there are no
  * more users of this io context, 0 otherwise.
@@ -3865,18 +3880,11 @@ int put_io_context(struct io_context *ioc)
        BUG_ON(atomic_read(&ioc->refcount) == 0);
 
        if (atomic_dec_and_test(&ioc->refcount)) {
-               struct cfq_io_context *cic;
-
                rcu_read_lock();
                if (ioc->aic && ioc->aic->dtor)
                        ioc->aic->dtor(ioc->aic);
-               if (ioc->cic_root.rb_node != NULL) {
-                       struct rb_node *n = rb_first(&ioc->cic_root);
-
-                       cic = rb_entry(n, struct cfq_io_context, rb_node);
-                       cic->dtor(ioc);
-               }
                rcu_read_unlock();
+               cfq_dtor(ioc);
 
                kmem_cache_free(iocontext_cachep, ioc);
                return 1;
@@ -3885,11 +3893,26 @@ int put_io_context(struct io_context *ioc)
 }
 EXPORT_SYMBOL(put_io_context);
 
+static void cfq_exit(struct io_context *ioc)
+{
+       struct cfq_io_context *cic[1];
+       int r;
+
+       rcu_read_lock();
+       /*
+        * See comment for cfq_dtor()
+        */
+       r = radix_tree_gang_lookup(&ioc->radix_root, (void **) cic, 0, 1);
+       rcu_read_unlock();
+
+       if (r > 0)
+               cic[0]->exit(ioc);
+}
+
 /* Called by the exitting task */
 void exit_io_context(void)
 {
        struct io_context *ioc;
-       struct cfq_io_context *cic;
 
        task_lock(current);
        ioc = current->io_context;
@@ -3899,11 +3922,7 @@ void exit_io_context(void)
        if (atomic_dec_and_test(&ioc->nr_tasks)) {
                if (ioc->aic && ioc->aic->exit)
                        ioc->aic->exit(ioc->aic);
-               if (ioc->cic_root.rb_node != NULL) {
-                       cic = rb_entry(rb_first(&ioc->cic_root),
-                               struct cfq_io_context, rb_node);
-                       cic->exit(ioc);
-               }
+               cfq_exit(ioc);
 
                put_io_context(ioc);
        }
@@ -3923,7 +3942,7 @@ struct io_context *alloc_io_context(gfp_t gfp_flags, int node)
                ret->last_waited = jiffies; /* doesn't matter... */
                ret->nr_batch_requests = 0; /* because this is 0 */
                ret->aic = NULL;
-               ret->cic_root.rb_node = NULL;
+               INIT_RADIX_TREE(&ret->radix_root, GFP_ATOMIC | __GFP_HIGH);
                ret->ioc_data = NULL;
        }