drivers/md/raid5.c

   1 /*
   2  * raid5.c : Multiple Devices driver for Linux
   3  *         Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman
   4  *         Copyright (C) 1999, 2000 Ingo Molnar
   5  *         Copyright (C) 2002, 2003 H. Peter Anvin
   6  *
   7  * RAID-4/5/6 management functions.
   8  * Thanks to Penguin Computing for making the RAID-6 development possible
   9  * by donating a test server!
  10  *
  11  * This program is free software; you can redistribute it and/or modify
  12  * it under the terms of the GNU General Public License as published by
  13  * the Free Software Foundation; either version 2, or (at your option)
  14  * any later version.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * (for example /usr/src/linux/COPYING); if not, write to the Free
  18  * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  19  */
  20
  21 /*
  22  * BITMAP UNPLUGGING:
  23  *
  24  * The sequencing for updating the bitmap reliably is a little
  25  * subtle (and I got it wrong the first time) so it deserves some
  26  * explanation.
  27  *
  28  * We group bitmap updates into batches.  Each batch has a number.
  29  * We may write out several batches at once, but that isn't very important.
  30  * conf->bm_write is the number of the last batch successfully written.
  31  * conf->bm_flush is the number of the last batch that was closed to
  32  *    new additions.
  33  * When we discover that we will need to write to any block in a stripe
  34  * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq
  35  * the number of the batch it will be in. This is bm_flush+1.
  36  * When we are ready to do a write, if that batch hasn't been written yet,
  37  *   we plug the array and queue the stripe for later.
  38  * When an unplug happens, we increment bm_flush, thus closing the current
  39  *   batch.
  40  * When we notice that bm_flush > bm_write, we write out all pending updates
  41  * to the bitmap, and advance bm_write to where bm_flush was.
  42  * This may occasionally write a bit out twice, but is sure never to
  43  * miss any bits.
  44  */
  45
  46 #include <linux/blkdev.h>
  47 #include <linux/kthread.h>
  48 #include <linux/raid/pq.h>
  49 #include <linux/async_tx.h>
  50 #include <linux/async.h>
  51 #include <linux/seq_file.h>
  52 #include <linux/cpu.h>
  53 #include "md.h"
  54 #include "raid5.h"
  55 #include "bitmap.h"
  56
  57 /*
  58  * Stripe cache
  59  */
  60
  61 #define NR_STRIPES              256
  62 #define STRIPE_SIZE             PAGE_SIZE
  63 #define STRIPE_SHIFT            (PAGE_SHIFT - 9)
  64 #define STRIPE_SECTORS          (STRIPE_SIZE>>9)
  65 #define IO_THRESHOLD            1
  66 #define BYPASS_THRESHOLD        1
  67 #define NR_HASH                 (PAGE_SIZE / sizeof(struct hlist_head))
  68 #define HASH_MASK               (NR_HASH - 1)
  69
  70 #define stripe_hash(conf, sect) (&((conf)->stripe_hashtbl[((sect) >> STRIPE_SHIFT) & HASH_MASK]))
  71
  72 /* bio's attached to a stripe+device for I/O are linked together in bi_sector
  73  * order without overlap.  There may be several bio's per stripe+device, and
  74  * a bio could span several devices.
  75  * When walking this list for a particular stripe+device, we must never proceed
  76  * beyond a bio that extends past this device, as the next bio might no longer
  77  * be valid.
  78  * This macro is used to determine the 'next' bio in the list, given the sector
  79  * of the current stripe+device
  80  */
  81 #define r5_next_bio(bio, sect) ( ( (bio)->bi_sector + ((bio)->bi_size>>9) < sect + STRIPE_SECTORS) ? (bio)->bi_next : NULL)
  82 /*
  83  * The following can be used to debug the driver
  84  */
  85 #define RAID5_PARANOIA  1
  86 #if RAID5_PARANOIA && defined(CONFIG_SMP)
  87 # define CHECK_DEVLOCK() assert_spin_locked(&conf->device_lock)
  88 #else
  89 # define CHECK_DEVLOCK()
  90 #endif
  91
  92 #ifdef DEBUG
  93 #define inline
  94 #define __inline__
  95 #endif
  96
  97 #define printk_rl(args...) ((void) (printk_ratelimit() && printk(args)))
  98
  99 /*
 100  * We maintain a biased count of active stripes in the bottom 16 bits of
 101  * bi_phys_segments, and a count of processed stripes in the upper 16 bits
 102  */
 103 static inline int raid5_bi_phys_segments(struct bio *bio)
 104 {
 105         return bio->bi_phys_segments & 0xffff;
 106 }
 107
 108 static inline int raid5_bi_hw_segments(struct bio *bio)
 109 {
 110         return (bio->bi_phys_segments >> 16) & 0xffff;
 111 }
 112
 113 static inline int raid5_dec_bi_phys_segments(struct bio *bio)
 114 {
 115         --bio->bi_phys_segments;
 116         return raid5_bi_phys_segments(bio);
 117 }
 118
 119 static inline int raid5_dec_bi_hw_segments(struct bio *bio)
 120 {
 121         unsigned short val = raid5_bi_hw_segments(bio);
 122
 123         --val;
 124         bio->bi_phys_segments = (val << 16) | raid5_bi_phys_segments(bio);
 125         return val;
 126 }
 127
 128 static inline void raid5_set_bi_hw_segments(struct bio *bio, unsigned int cnt)
 129 {
 130         bio->bi_phys_segments = raid5_bi_phys_segments(bio) || (cnt << 16);
 131 }
 132
 133 /* Find first data disk in a raid6 stripe */
 134 static inline int raid6_d0(struct stripe_head *sh)
 135 {
 136         if (sh->ddf_layout)
 137                 /* ddf always start from first device */
 138                 return 0;
 139         /* md starts just after Q block */
 140         if (sh->qd_idx == sh->disks - 1)
 141                 return 0;
 142         else
 143                 return sh->qd_idx + 1;
 144 }
 145 static inline int raid6_next_disk(int disk, int raid_disks)
 146 {
 147         disk++;
 148         return (disk < raid_disks) ? disk : 0;
 149 }
 150
 151 /* When walking through the disks in a raid5, starting at raid6_d0,
 152  * We need to map each disk to a 'slot', where the data disks are slot
 153  * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk
 154  * is raid_disks-1.  This help does that mapping.
 155  */
 156 static int raid6_idx_to_slot(int idx, struct stripe_head *sh,
 157                              int *count, int syndrome_disks)
 158 {
 159         int slot;
 160
 161         if (idx == sh->pd_idx)
 162                 return syndrome_disks;
 163         if (idx == sh->qd_idx)
 164                 return syndrome_disks + 1;
 165         slot = (*count)++;
 166         return slot;
 167 }
 168
 169 static void return_io(struct bio *return_bi)
 170 {
 171         struct bio *bi = return_bi;
 172         while (bi) {
 173
 174                 return_bi = bi->bi_next;
 175                 bi->bi_next = NULL;
 176                 bi->bi_size = 0;
 177                 bio_endio(bi, 0);
 178                 bi = return_bi;
 179         }
 180 }
 181
 182 static void print_raid5_conf (raid5_conf_t *conf);
 183
 184 static int stripe_operations_active(struct stripe_head *sh)
 185 {
 186         return sh->check_state || sh->reconstruct_state ||
 187                test_bit(STRIPE_BIOFILL_RUN, &sh->state) ||
 188                test_bit(STRIPE_COMPUTE_RUN, &sh->state);
 189 }
 190
 191 static void __release_stripe(raid5_conf_t *conf, struct stripe_head *sh)
 192 {
 193         if (atomic_dec_and_test(&sh->count)) {
 194                 BUG_ON(!list_empty(&sh->lru));
 195                 BUG_ON(atomic_read(&conf->active_stripes)==0);
 196                 if (test_bit(STRIPE_HANDLE, &sh->state)) {
 197                         if (test_bit(STRIPE_DELAYED, &sh->state)) {
 198                                 list_add_tail(&sh->lru, &conf->delayed_list);
 199                                 blk_plug_device(conf->mddev->queue);
 200                         } else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&
 201                                    sh->bm_seq - conf->seq_write > 0) {
 202                                 list_add_tail(&sh->lru, &conf->bitmap_list);
 203                                 blk_plug_device(conf->mddev->queue);
 204                         } else {
 205                                 clear_bit(STRIPE_BIT_DELAY, &sh->state);
 206                                 list_add_tail(&sh->lru, &conf->handle_list);
 207                         }
 208                         md_wakeup_thread(conf->mddev->thread);
 209                 } else {
 210                         BUG_ON(stripe_operations_active(sh));
 211                         if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
 212                                 atomic_dec(&conf->preread_active_stripes);
 213                                 if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD)
 214                                         md_wakeup_thread(conf->mddev->thread);
 215                         }
 216                         atomic_dec(&conf->active_stripes);
 217                         if (!test_bit(STRIPE_EXPANDING, &sh->state)) {
 218                                 list_add_tail(&sh->lru, &conf->inactive_list);
 219                                 wake_up(&conf->wait_for_stripe);
 220                                 if (conf->retry_read_aligned)
 221                                         md_wakeup_thread(conf->mddev->thread);
 222                         }
 223                 }
 224         }
 225 }
 226
 227 static void release_stripe(struct stripe_head *sh)
 228 {
 229         raid5_conf_t *conf = sh->raid_conf;
 230         unsigned long flags;
 231
 232         spin_lock_irqsave(&conf->device_lock, flags);
 233         __release_stripe(conf, sh);
 234         spin_unlock_irqrestore(&conf->device_lock, flags);
 235 }
 236
 237 static inline void remove_hash(struct stripe_head *sh)
 238 {
 239         pr_debug("remove_hash(), stripe %llu\n",
 240                 (unsigned long long)sh->sector);
 241
 242         hlist_del_init(&sh->hash);
 243 }
 244
 245 static inline void insert_hash(raid5_conf_t *conf, struct stripe_head *sh)
 246 {
 247         struct hlist_head *hp = stripe_hash(conf, sh->sector);
 248
 249         pr_debug("insert_hash(), stripe %llu\n",
 250                 (unsigned long long)sh->sector);
 251
 252         CHECK_DEVLOCK();
 253         hlist_add_head(&sh->hash, hp);
 254 }
 255
 256
 257 /* find an idle stripe, make sure it is unhashed, and return it. */
 258 static struct stripe_head *get_free_stripe(raid5_conf_t *conf)
 259 {
 260         struct stripe_head *sh = NULL;
 261         struct list_head *first;
 262
 263         CHECK_DEVLOCK();
 264         if (list_empty(&conf->inactive_list))
 265                 goto out;
 266         first = conf->inactive_list.next;
 267         sh = list_entry(first, struct stripe_head, lru);
 268         list_del_init(first);
 269         remove_hash(sh);
 270         atomic_inc(&conf->active_stripes);
 271 out:
 272         return sh;
 273 }
 274
 275 static void shrink_buffers(struct stripe_head *sh, int num)
 276 {
 277         struct page *p;
 278         int i;
 279
 280         for (i=0; i<num ; i++) {
 281                 p = sh->dev[i].page;
 282                 if (!p)
 283                         continue;
 284                 sh->dev[i].page = NULL;
 285                 put_page(p);
 286         }
 287 }
 288
 289 static int grow_buffers(struct stripe_head *sh, int num)
 290 {
 291         int i;
 292
 293         for (i=0; i<num; i++) {
 294                 struct page *page;
 295
 296                 if (!(page = alloc_page(GFP_KERNEL))) {
 297                         return 1;
 298                 }
 299                 sh->dev[i].page = page;
 300         }
 301         return 0;
 302 }
 303
 304 static void raid5_build_block(struct stripe_head *sh, int i, int previous);
 305 static void stripe_set_idx(sector_t stripe, raid5_conf_t *conf, int previous,
 306                             struct stripe_head *sh);
 307
 308 static void init_stripe(struct stripe_head *sh, sector_t sector, int previous)
 309 {
 310         raid5_conf_t *conf = sh->raid_conf;
 311         int i;
 312
 313         BUG_ON(atomic_read(&sh->count) != 0);
 314         BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));
 315         BUG_ON(stripe_operations_active(sh));
 316
 317         CHECK_DEVLOCK();
 318         pr_debug("init_stripe called, stripe %llu\n",
 319                 (unsigned long long)sh->sector);
 320
 321         remove_hash(sh);
 322
 323         sh->generation = conf->generation - previous;
 324         sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks;
 325         sh->sector = sector;
 326         stripe_set_idx(sector, conf, previous, sh);
 327         sh->state = 0;
 328
 329
 330         for (i = sh->disks; i--; ) {
 331                 struct r5dev *dev = &sh->dev[i];
 332
 333                 if (dev->toread || dev->read || dev->towrite || dev->written ||
 334                     test_bit(R5_LOCKED, &dev->flags)) {
 335                         printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n",
 336                                (unsigned long long)sh->sector, i, dev->toread,
 337                                dev->read, dev->towrite, dev->written,
 338                                test_bit(R5_LOCKED, &dev->flags));
 339                         BUG();
 340                 }
 341                 dev->flags = 0;
 342                 raid5_build_block(sh, i, previous);
 343         }
 344         insert_hash(conf, sh);
 345 }
 346
 347 static struct stripe_head *__find_stripe(raid5_conf_t *conf, sector_t sector,
 348                                          short generation)
 349 {
 350         struct stripe_head *sh;
 351         struct hlist_node *hn;
 352
 353         CHECK_DEVLOCK();
 354         pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector);
 355         hlist_for_each_entry(sh, hn, stripe_hash(conf, sector), hash)
 356                 if (sh->sector == sector && sh->generation == generation)
 357                         return sh;
 358         pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector);
 359         return NULL;
 360 }
 361
 362 static void unplug_slaves(mddev_t *mddev);
 363 static void raid5_unplug_device(struct request_queue *q);
 364
 365 static struct stripe_head *
 366 get_active_stripe(raid5_conf_t *conf, sector_t sector,
 367                   int previous, int noblock, int noquiesce)
 368 {
 369         struct stripe_head *sh;
 370
 371         pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector);
 372
 373         spin_lock_irq(&conf->device_lock);
 374
 375         do {
 376                 wait_event_lock_irq(conf->wait_for_stripe,
 377                                     conf->quiesce == 0 || noquiesce,
 378                                     conf->device_lock, /* nothing */);
 379                 sh = __find_stripe(conf, sector, conf->generation - previous);
 380                 if (!sh) {
 381                         if (!conf->inactive_blocked)
 382                                 sh = get_free_stripe(conf);
 383                         if (noblock && sh == NULL)
 384                                 break;
 385                         if (!sh) {
 386                                 conf->inactive_blocked = 1;
 387                                 wait_event_lock_irq(conf->wait_for_stripe,
 388                                                     !list_empty(&conf->inactive_list) &&
 389                                                     (atomic_read(&conf->active_stripes)
 390                                                      < (conf->max_nr_stripes *3/4)
 391                                                      || !conf->inactive_blocked),
 392                                                     conf->device_lock,
 393                                                     raid5_unplug_device(conf->mddev->queue)
 394                                         );
 395                                 conf->inactive_blocked = 0;
 396                         } else
 397                                 init_stripe(sh, sector, previous);
 398                 } else {
 399                         if (atomic_read(&sh->count)) {
 400                                 BUG_ON(!list_empty(&sh->lru)
 401                                     && !test_bit(STRIPE_EXPANDING, &sh->state));
 402                         } else {
 403                                 if (!test_bit(STRIPE_HANDLE, &sh->state))
 404                                         atomic_inc(&conf->active_stripes);
 405                                 if (list_empty(&sh->lru) &&
 406                                     !test_bit(STRIPE_EXPANDING, &sh->state))
 407                                         BUG();
 408                                 list_del_init(&sh->lru);
 409                         }
 410                 }
 411         } while (sh == NULL);
 412
 413         if (sh)
 414                 atomic_inc(&sh->count);
 415
 416         spin_unlock_irq(&conf->device_lock);
 417         return sh;
 418 }
 419
 420 static void
 421 raid5_end_read_request(struct bio *bi, int error);
 422 static void
 423 raid5_end_write_request(struct bio *bi, int error);
 424
 425 static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
 426 {
 427         raid5_conf_t *conf = sh->raid_conf;
 428         int i, disks = sh->disks;
 429
 430         might_sleep();
 431
 432         for (i = disks; i--; ) {
 433                 int rw;
 434                 struct bio *bi;
 435                 mdk_rdev_t *rdev;
 436                 if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags))
 437                         rw = WRITE;
 438                 else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))
 439                         rw = READ;
 440                 else
 441                         continue;
 442
 443                 bi = &sh->dev[i].req;
 444
 445                 bi->bi_rw = rw;
 446                 if (rw == WRITE)
 447                         bi->bi_end_io = raid5_end_write_request;
 448                 else
 449                         bi->bi_end_io = raid5_end_read_request;
 450
 451                 rcu_read_lock();
 452                 rdev = rcu_dereference(conf->disks[i].rdev);
 453                 if (rdev && test_bit(Faulty, &rdev->flags))
 454                         rdev = NULL;
 455                 if (rdev)
 456                         atomic_inc(&rdev->nr_pending);
 457                 rcu_read_unlock();
 458
 459                 if (rdev) {
 460                         if (s->syncing || s->expanding || s->expanded)
 461                                 md_sync_acct(rdev->bdev, STRIPE_SECTORS);
 462
 463                         set_bit(STRIPE_IO_STARTED, &sh->state);
 464
 465                         bi->bi_bdev = rdev->bdev;
 466                         pr_debug("%s: for %llu schedule op %ld on disc %d\n",
 467                                 __func__, (unsigned long long)sh->sector,
 468                                 bi->bi_rw, i);
 469                         atomic_inc(&sh->count);
 470                         bi->bi_sector = sh->sector + rdev->data_offset;
 471                         bi->bi_flags = 1 << BIO_UPTODATE;
 472                         bi->bi_vcnt = 1;
 473                         bi->bi_max_vecs = 1;
 474                         bi->bi_idx = 0;
 475                         bi->bi_io_vec = &sh->dev[i].vec;
 476                         bi->bi_io_vec[0].bv_len = STRIPE_SIZE;
 477                         bi->bi_io_vec[0].bv_offset = 0;
 478                         bi->bi_size = STRIPE_SIZE;
 479                         bi->bi_next = NULL;
 480                         if (rw == WRITE &&
 481                             test_bit(R5_ReWrite, &sh->dev[i].flags))
 482                                 atomic_add(STRIPE_SECTORS,
 483                                         &rdev->corrected_errors);
 484                         generic_make_request(bi);
 485                 } else {
 486                         if (rw == WRITE)
 487                                 set_bit(STRIPE_DEGRADED, &sh->state);
 488                         pr_debug("skip op %ld on disc %d for sector %llu\n",
 489                                 bi->bi_rw, i, (unsigned long long)sh->sector);
 490                         clear_bit(R5_LOCKED, &sh->dev[i].flags);
 491                         set_bit(STRIPE_HANDLE, &sh->state);
 492                 }
 493         }
 494 }
 495
 496 static struct dma_async_tx_descriptor *
 497 async_copy_data(int frombio, struct bio *bio, struct page *page,
 498         sector_t sector, struct dma_async_tx_descriptor *tx)
 499 {
 500         struct bio_vec *bvl;
 501         struct page *bio_page;
 502         int i;
 503         int page_offset;
 504         struct async_submit_ctl submit;
 505         enum async_tx_flags flags = 0;
 506
 507         if (bio->bi_sector >= sector)
 508                 page_offset = (signed)(bio->bi_sector - sector) * 512;
 509         else
 510                 page_offset = (signed)(sector - bio->bi_sector) * -512;
 511
 512         if (frombio)
 513                 flags |= ASYNC_TX_FENCE;
 514         init_async_submit(&submit, flags, tx, NULL, NULL, NULL);
 515
 516         bio_for_each_segment(bvl, bio, i) {
 517                 int len = bio_iovec_idx(bio, i)->bv_len;
 518                 int clen;
 519                 int b_offset = 0;
 520
 521                 if (page_offset < 0) {
 522                         b_offset = -page_offset;
 523                         page_offset += b_offset;
 524                         len -= b_offset;
 525                 }
 526
 527                 if (len > 0 && page_offset + len > STRIPE_SIZE)
 528                         clen = STRIPE_SIZE - page_offset;
 529                 else
 530                         clen = len;
 531
 532                 if (clen > 0) {
 533                         b_offset += bio_iovec_idx(bio, i)->bv_offset;
 534                         bio_page = bio_iovec_idx(bio, i)->bv_page;
 535                         if (frombio)
 536                                 tx = async_memcpy(page, bio_page, page_offset,
 537                                                   b_offset, clen, &submit);
 538                         else
 539                                 tx = async_memcpy(bio_page, page, b_offset,
 540                                                   page_offset, clen, &submit);
 541                 }
 542                 /* chain the operations */
 543                 submit.depend_tx = tx;
 544
 545                 if (clen < len) /* hit end of page */
 546                         break;
 547                 page_offset +=  len;
 548         }
 549
 550         return tx;
 551 }
 552
 553 static void ops_complete_biofill(void *stripe_head_ref)
 554 {
 555         struct stripe_head *sh = stripe_head_ref;
 556         struct bio *return_bi = NULL;
 557         raid5_conf_t *conf = sh->raid_conf;
 558         int i;
 559
 560         pr_debug("%s: stripe %llu\n", __func__,
 561                 (unsigned long long)sh->sector);
 562
 563         /* clear completed biofills */
 564         spin_lock_irq(&conf->device_lock);
 565         for (i = sh->disks; i--; ) {
 566                 struct r5dev *dev = &sh->dev[i];
 567
 568                 /* acknowledge completion of a biofill operation */
 569                 /* and check if we need to reply to a read request,
 570                  * new R5_Wantfill requests are held off until
 571                  * !STRIPE_BIOFILL_RUN
 572                  */
 573                 if (test_and_clear_bit(R5_Wantfill, &dev->flags)) {
 574                         struct bio *rbi, *rbi2;
 575
 576                         BUG_ON(!dev->read);
 577                         rbi = dev->read;
 578                         dev->read = NULL;
 579                         while (rbi && rbi->bi_sector <
 580                                 dev->sector + STRIPE_SECTORS) {
 581                                 rbi2 = r5_next_bio(rbi, dev->sector);
 582                                 if (!raid5_dec_bi_phys_segments(rbi)) {
 583                                         rbi->bi_next = return_bi;
 584                                         return_bi = rbi;
 585                                 }
 586                                 rbi = rbi2;
 587                         }
 588                 }
 589         }
 590         spin_unlock_irq(&conf->device_lock);
 591         clear_bit(STRIPE_BIOFILL_RUN, &sh->state);
 592
 593         return_io(return_bi);
 594
 595         set_bit(STRIPE_HANDLE, &sh->state);
 596         release_stripe(sh);
 597 }
 598
 599 static void ops_run_biofill(struct stripe_head *sh)
 600 {
 601         struct dma_async_tx_descriptor *tx = NULL;
 602         raid5_conf_t *conf = sh->raid_conf;
 603         struct async_submit_ctl submit;
 604         int i;
 605
 606         pr_debug("%s: stripe %llu\n", __func__,
 607                 (unsigned long long)sh->sector);
 608
 609         for (i = sh->disks; i--; ) {
 610                 struct r5dev *dev = &sh->dev[i];
 611                 if (test_bit(R5_Wantfill, &dev->flags)) {
 612                         struct bio *rbi;
 613                         spin_lock_irq(&conf->device_lock);
 614                         dev->read = rbi = dev->toread;
 615                         dev->toread = NULL;
 616                         spin_unlock_irq(&conf->device_lock);
 617                         while (rbi && rbi->bi_sector <
 618                                 dev->sector + STRIPE_SECTORS) {
 619                                 tx = async_copy_data(0, rbi, dev->page,
 620                                         dev->sector, tx);
 621                                 rbi = r5_next_bio(rbi, dev->sector);
 622                         }
 623                 }
 624         }
 625
 626         atomic_inc(&sh->count);
 627         init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL);
 628         async_trigger_callback(&submit);
 629 }
 630
 631 static void mark_target_uptodate(struct stripe_head *sh, int target)
 632 {
 633         struct r5dev *tgt;
 634
 635         if (target < 0)
 636                 return;
 637
 638         tgt = &sh->dev[target];
 639         set_bit(R5_UPTODATE, &tgt->flags);
 640         BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
 641         clear_bit(R5_Wantcompute, &tgt->flags);
 642 }
 643
 644 static void ops_complete_compute(void *stripe_head_ref)
 645 {
 646         struct stripe_head *sh = stripe_head_ref;
 647
 648         pr_debug("%s: stripe %llu\n", __func__,
 649                 (unsigned long long)sh->sector);
 650
 651         /* mark the computed target(s) as uptodate */
 652         mark_target_uptodate(sh, sh->ops.target);
 653         mark_target_uptodate(sh, sh->ops.target2);
 654
 655         clear_bit(STRIPE_COMPUTE_RUN, &sh->state);
 656         if (sh->check_state == check_state_compute_run)
 657                 sh->check_state = check_state_compute_result;
 658         set_bit(STRIPE_HANDLE, &sh->state);
 659         release_stripe(sh);
 660 }
 661
 662 /* return a pointer to the address conversion region of the scribble buffer */
 663 static addr_conv_t *to_addr_conv(struct stripe_head *sh,
 664                                  struct raid5_percpu *percpu)
 665 {
 666         return percpu->scribble + sizeof(struct page *) * (sh->disks + 2);
 667 }
 668
 669 static struct dma_async_tx_descriptor *
 670 ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
 671 {
 672         int disks = sh->disks;
 673         struct page **xor_srcs = percpu->scribble;
 674         int target = sh->ops.target;
 675         struct r5dev *tgt = &sh->dev[target];
 676         struct page *xor_dest = tgt->page;
 677         int count = 0;
 678         struct dma_async_tx_descriptor *tx;
 679         struct async_submit_ctl submit;
 680         int i;
 681
 682         pr_debug("%s: stripe %llu block: %d\n",
 683                 __func__, (unsigned long long)sh->sector, target);
 684         BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
 685
 686         for (i = disks; i--; )
 687                 if (i != target)
 688                         xor_srcs[count++] = sh->dev[i].page;
 689
 690         atomic_inc(&sh->count);
 691
 692         init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL,
 693                           ops_complete_compute, sh, to_addr_conv(sh, percpu));
 694         if (unlikely(count == 1))
 695                 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
 696         else
 697                 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
 698
 699         return tx;
 700 }
 701
 702 /* set_syndrome_sources - populate source buffers for gen_syndrome
 703  * @srcs - (struct page *) array of size sh->disks
 704  * @sh - stripe_head to parse
 705  *
 706  * Populates srcs in proper layout order for the stripe and returns the
 707  * 'count' of sources to be used in a call to async_gen_syndrome.  The P
 708  * destination buffer is recorded in srcs[count] and the Q destination
 709  * is recorded in srcs[count+1]].
 710  */
 711 static int set_syndrome_sources(struct page **srcs, struct stripe_head *sh)
 712 {
 713         int disks = sh->disks;
 714         int syndrome_disks = sh->ddf_layout ? disks : (disks - 2);
 715         int d0_idx = raid6_d0(sh);
 716         int count;
 717         int i;
 718
 719         for (i = 0; i < disks; i++)
 720                 srcs[i] = (void *)raid6_empty_zero_page;
 721
 722         count = 0;
 723         i = d0_idx;
 724         do {
 725                 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
 726
 727                 srcs[slot] = sh->dev[i].page;
 728                 i = raid6_next_disk(i, disks);
 729         } while (i != d0_idx);
 730         BUG_ON(count != syndrome_disks);
 731
 732         return count;
 733 }
 734
 735 static struct dma_async_tx_descriptor *
 736 ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
 737 {
 738         int disks = sh->disks;
 739         struct page **blocks = percpu->scribble;
 740         int target;
 741         int qd_idx = sh->qd_idx;
 742         struct dma_async_tx_descriptor *tx;
 743         struct async_submit_ctl submit;
 744         struct r5dev *tgt;
 745         struct page *dest;
 746         int i;
 747         int count;
 748
 749         if (sh->ops.target < 0)
 750                 target = sh->ops.target2;
 751         else if (sh->ops.target2 < 0)
 752                 target = sh->ops.target;
 753         else
 754                 /* we should only have one valid target */
 755                 BUG();
 756         BUG_ON(target < 0);
 757         pr_debug("%s: stripe %llu block: %d\n",
 758                 __func__, (unsigned long long)sh->sector, target);
 759
 760         tgt = &sh->dev[target];
 761         BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
 762         dest = tgt->page;
 763
 764         atomic_inc(&sh->count);
 765
 766         if (target == qd_idx) {
 767                 count = set_syndrome_sources(blocks, sh);
 768                 blocks[count] = NULL; /* regenerating p is not necessary */
 769                 BUG_ON(blocks[count+1] != dest); /* q should already be set */
 770                 init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
 771                                   ops_complete_compute, sh,
 772                                   to_addr_conv(sh, percpu));
 773                 tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
 774         } else {
 775                 /* Compute any data- or p-drive using XOR */
 776                 count = 0;
 777                 for (i = disks; i-- ; ) {
 778                         if (i == target || i == qd_idx)
 779                                 continue;
 780                         blocks[count++] = sh->dev[i].page;
 781                 }
 782
 783                 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
 784                                   NULL, ops_complete_compute, sh,
 785                                   to_addr_conv(sh, percpu));
 786                 tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit);
 787         }
 788
 789         return tx;
 790 }
 791
 792 static struct dma_async_tx_descriptor *
 793 ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
 794 {
 795         int i, count, disks = sh->disks;
 796         int syndrome_disks = sh->ddf_layout ? disks : disks-2;
 797         int d0_idx = raid6_d0(sh);
 798         int faila = -1, failb = -1;
 799         int target = sh->ops.target;
 800         int target2 = sh->ops.target2;
 801         struct r5dev *tgt = &sh->dev[target];
 802         struct r5dev *tgt2 = &sh->dev[target2];
 803         struct dma_async_tx_descriptor *tx;
 804         struct page **blocks = percpu->scribble;
 805         struct async_submit_ctl submit;
 806
 807         pr_debug("%s: stripe %llu block1: %d block2: %d\n",
 808                  __func__, (unsigned long long)sh->sector, target, target2);
 809         BUG_ON(target < 0 || target2 < 0);
 810         BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
 811         BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags));
 812
 813         /* we need to open-code set_syndrome_sources to handle to the
 814          * slot number conversion for 'faila' and 'failb'
 815          */
 816         for (i = 0; i < disks ; i++)
 817                 blocks[i] = (void *)raid6_empty_zero_page;
 818         count = 0;
 819         i = d0_idx;
 820         do {
 821                 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
 822
 823                 blocks[slot] = sh->dev[i].page;
 824
 825                 if (i == target)
 826                         faila = slot;
 827                 if (i == target2)
 828                         failb = slot;
 829                 i = raid6_next_disk(i, disks);
 830         } while (i != d0_idx);
 831         BUG_ON(count != syndrome_disks);
 832
 833         BUG_ON(faila == failb);
 834         if (failb < faila)
 835                 swap(faila, failb);
 836         pr_debug("%s: stripe: %llu faila: %d failb: %d\n",
 837                  __func__, (unsigned long long)sh->sector, faila, failb);
 838
 839         atomic_inc(&sh->count);
 840
 841         if (failb == syndrome_disks+1) {
 842                 /* Q disk is one of the missing disks */
 843                 if (faila == syndrome_disks) {
 844                         /* Missing P+Q, just recompute */
 845                         init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
 846                                           ops_complete_compute, sh,
 847                                           to_addr_conv(sh, percpu));
 848                         return async_gen_syndrome(blocks, 0, count+2,
 849                                                   STRIPE_SIZE, &submit);
 850                 } else {
 851                         struct page *dest;
 852                         int data_target;
 853                         int qd_idx = sh->qd_idx;
 854
 855                         /* Missing D+Q: recompute D from P, then recompute Q */
 856                         if (target == qd_idx)
 857                                 data_target = target2;
 858                         else
 859                                 data_target = target;
 860
 861                         count = 0;
 862                         for (i = disks; i-- ; ) {
 863                                 if (i == data_target || i == qd_idx)
 864                                         continue;
 865                                 blocks[count++] = sh->dev[i].page;
 866                         }
 867                         dest = sh->dev[data_target].page;
 868                         init_async_submit(&submit,
 869                                           ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
 870                                           NULL, NULL, NULL,
 871                                           to_addr_conv(sh, percpu));
 872                         tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE,
 873                                        &submit);
 874
 875                         count = set_syndrome_sources(blocks, sh);
 876                         init_async_submit(&submit, ASYNC_TX_FENCE, tx,
 877                                           ops_complete_compute, sh,
 878                                           to_addr_conv(sh, percpu));
 879                         return async_gen_syndrome(blocks, 0, count+2,
 880                                                   STRIPE_SIZE, &submit);
 881                 }
 882         }
 883
 884         init_async_submit(&submit, ASYNC_TX_FENCE, NULL, ops_complete_compute,
 885                           sh, to_addr_conv(sh, percpu));
 886         if (failb == syndrome_disks) {
 887                 /* We're missing D+P. */
 888                 return async_raid6_datap_recov(syndrome_disks+2, STRIPE_SIZE,
 889                                                faila, blocks, &submit);
 890         } else {
 891                 /* We're missing D+D. */
 892                 return async_raid6_2data_recov(syndrome_disks+2, STRIPE_SIZE,
 893                                                faila, failb, blocks, &submit);
 894         }
 895 }
 896
 897
 898 static void ops_complete_prexor(void *stripe_head_ref)
 899 {
 900         struct stripe_head *sh = stripe_head_ref;
 901
 902         pr_debug("%s: stripe %llu\n", __func__,
 903                 (unsigned long long)sh->sector);
 904 }
 905
 906 static struct dma_async_tx_descriptor *
 907 ops_run_prexor(struct stripe_head *sh, struct raid5_percpu *percpu,
 908                struct dma_async_tx_descriptor *tx)
 909 {
 910         int disks = sh->disks;
 911         struct page **xor_srcs = percpu->scribble;
 912         int count = 0, pd_idx = sh->pd_idx, i;
 913         struct async_submit_ctl submit;
 914
 915         /* existing parity data subtracted */
 916         struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
 917
 918         pr_debug("%s: stripe %llu\n", __func__,
 919                 (unsigned long long)sh->sector);
 920
 921         for (i = disks; i--; ) {
 922                 struct r5dev *dev = &sh->dev[i];
 923                 /* Only process blocks that are known to be uptodate */
 924                 if (test_bit(R5_Wantdrain, &dev->flags))
 925                         xor_srcs[count++] = dev->page;
 926         }
 927
 928         init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
 929                           ops_complete_prexor, sh, to_addr_conv(sh, percpu));
 930         tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
 931
 932         return tx;
 933 }
 934
 935 static struct dma_async_tx_descriptor *
 936 ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
 937 {
 938         int disks = sh->disks;
 939         int i;
 940
 941         pr_debug("%s: stripe %llu\n", __func__,
 942                 (unsigned long long)sh->sector);
 943
 944         for (i = disks; i--; ) {
 945                 struct r5dev *dev = &sh->dev[i];
 946                 struct bio *chosen;
 947
 948                 if (test_and_clear_bit(R5_Wantdrain, &dev->flags)) {
 949                         struct bio *wbi;
 950
 951                         spin_lock(&sh->lock);
 952                         chosen = dev->towrite;
 953                         dev->towrite = NULL;
 954                         BUG_ON(dev->written);
 955                         wbi = dev->written = chosen;
 956                         spin_unlock(&sh->lock);
 957
 958                         while (wbi && wbi->bi_sector <
 959                                 dev->sector + STRIPE_SECTORS) {
 960                                 tx = async_copy_data(1, wbi, dev->page,
 961                                         dev->sector, tx);
 962                                 wbi = r5_next_bio(wbi, dev->sector);
 963                         }
 964                 }
 965         }
 966
 967         return tx;
 968 }
 969
 970 static void ops_complete_reconstruct(void *stripe_head_ref)
 971 {
 972         struct stripe_head *sh = stripe_head_ref;
 973         int disks = sh->disks;
 974         int pd_idx = sh->pd_idx;
 975         int qd_idx = sh->qd_idx;
 976         int i;
 977
 978         pr_debug("%s: stripe %llu\n", __func__,
 979                 (unsigned long long)sh->sector);
 980
 981         for (i = disks; i--; ) {
 982                 struct r5dev *dev = &sh->dev[i];
 983
 984                 if (dev->written || i == pd_idx || i == qd_idx)
 985                         set_bit(R5_UPTODATE, &dev->flags);
 986         }
 987
 988         if (sh->reconstruct_state == reconstruct_state_drain_run)
 989                 sh->reconstruct_state = reconstruct_state_drain_result;
 990         else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run)
 991                 sh->reconstruct_state = reconstruct_state_prexor_drain_result;
 992         else {
 993                 BUG_ON(sh->reconstruct_state != reconstruct_state_run);
 994                 sh->reconstruct_state = reconstruct_state_result;
 995         }
 996
 997         set_bit(STRIPE_HANDLE, &sh->state);
 998         release_stripe(sh);
 999 }
1000
1001 static void
1002 ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,
1003                      struct dma_async_tx_descriptor *tx)
1004 {
1005         int disks = sh->disks;
1006         struct page **xor_srcs = percpu->scribble;
1007         struct async_submit_ctl submit;
1008         int count = 0, pd_idx = sh->pd_idx, i;
1009         struct page *xor_dest;
1010         int prexor = 0;
1011         unsigned long flags;
1012
1013         pr_debug("%s: stripe %llu\n", __func__,
1014                 (unsigned long long)sh->sector);
1015
1016         /* check if prexor is active which means only process blocks
1017          * that are part of a read-modify-write (written)
1018          */
1019         if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
1020                 prexor = 1;
1021                 xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
1022                 for (i = disks; i--; ) {
1023                         struct r5dev *dev = &sh->dev[i];
1024                         if (dev->written)
1025                                 xor_srcs[count++] = dev->page;
1026                 }
1027         } else {
1028                 xor_dest = sh->dev[pd_idx].page;
1029                 for (i = disks; i--; ) {
1030                         struct r5dev *dev = &sh->dev[i];
1031                         if (i != pd_idx)
1032                                 xor_srcs[count++] = dev->page;
1033                 }
1034         }
1035
1036         /* 1/ if we prexor'd then the dest is reused as a source
1037          * 2/ if we did not prexor then we are redoing the parity
1038          * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST
1039          * for the synchronous xor case
1040          */
1041         flags = ASYNC_TX_ACK |
1042                 (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST);
1043
1044         atomic_inc(&sh->count);
1045
1046         init_async_submit(&submit, flags, tx, ops_complete_reconstruct, sh,
1047                           to_addr_conv(sh, percpu));
1048         if (unlikely(count == 1))
1049                 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
1050         else
1051                 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
1052 }
1053
1054 static void
1055 ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
1056                      struct dma_async_tx_descriptor *tx)
1057 {
1058         struct async_submit_ctl submit;
1059         struct page **blocks = percpu->scribble;
1060         int count;
1061
1062         pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);
1063
1064         count = set_syndrome_sources(blocks, sh);
1065
1066         atomic_inc(&sh->count);
1067
1068         init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_reconstruct,
1069                           sh, to_addr_conv(sh, percpu));
1070         async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE,  &submit);
1071 }
1072
1073 static void ops_complete_check(void *stripe_head_ref)
1074 {
1075         struct stripe_head *sh = stripe_head_ref;
1076
1077         pr_debug("%s: stripe %llu\n", __func__,
1078                 (unsigned long long)sh->sector);
1079
1080         sh->check_state = check_state_check_result;
1081         set_bit(STRIPE_HANDLE, &sh->state);
1082         release_stripe(sh);
1083 }
1084
1085 static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)
1086 {
1087         int disks = sh->disks;
1088         int pd_idx = sh->pd_idx;
1089         int qd_idx = sh->qd_idx;
1090         struct page *xor_dest;
1091         struct page **xor_srcs = percpu->scribble;
1092         struct dma_async_tx_descriptor *tx;
1093         struct async_submit_ctl submit;
1094         int count;
1095         int i;
1096
1097         pr_debug("%s: stripe %llu\n", __func__,
1098                 (unsigned long long)sh->sector);
1099
1100         count = 0;
1101         xor_dest = sh->dev[pd_idx].page;
1102         xor_srcs[count++] = xor_dest;
1103         for (i = disks; i--; ) {
1104                 if (i == pd_idx || i == qd_idx)
1105                         continue;
1106                 xor_srcs[count++] = sh->dev[i].page;
1107         }
1108
1109         init_async_submit(&submit, 0, NULL, NULL, NULL,
1110                           to_addr_conv(sh, percpu));
1111         tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE,
1112                            &sh->ops.zero_sum_result, &submit);
1113
1114         atomic_inc(&sh->count);
1115         init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL);
1116         tx = async_trigger_callback(&submit);
1117 }
1118
1119 static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp)
1120 {
1121         struct page **srcs = percpu->scribble;
1122         struct async_submit_ctl submit;
1123         int count;
1124
1125         pr_debug("%s: stripe %llu checkp: %d\n", __func__,
1126                 (unsigned long long)sh->sector, checkp);
1127
1128         count = set_syndrome_sources(srcs, sh);
1129         if (!checkp)
1130                 srcs[count] = NULL;
1131
1132         atomic_inc(&sh->count);
1133         init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check,
1134                           sh, to_addr_conv(sh, percpu));
1135         async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE,
1136                            &sh->ops.zero_sum_result, percpu->spare_page, &submit);
1137 }
1138
1139 static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
1140 {
1141         int overlap_clear = 0, i, disks = sh->disks;
1142         struct dma_async_tx_descriptor *tx = NULL;
1143         raid5_conf_t *conf = sh->raid_conf;
1144         int level = conf->level;
1145         struct raid5_percpu *percpu;
1146         unsigned long cpu;
1147
1148         cpu = get_cpu();
1149         percpu = per_cpu_ptr(conf->percpu, cpu);
1150         if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) {
1151                 ops_run_biofill(sh);
1152                 overlap_clear++;
1153         }
1154
1155         if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) {
1156                 if (level < 6)
1157                         tx = ops_run_compute5(sh, percpu);
1158                 else {
1159                         if (sh->ops.target2 < 0 || sh->ops.target < 0)
1160                                 tx = ops_run_compute6_1(sh, percpu);
1161                         else
1162                                 tx = ops_run_compute6_2(sh, percpu);
1163                 }
1164                 /* terminate the chain if reconstruct is not set to be run */
1165                 if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request))
1166                         async_tx_ack(tx);
1167         }
1168
1169         if (test_bit(STRIPE_OP_PREXOR, &ops_request))
1170                 tx = ops_run_prexor(sh, percpu, tx);
1171
1172         if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) {
1173                 tx = ops_run_biodrain(sh, tx);
1174                 overlap_clear++;
1175         }
1176
1177         if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) {
1178                 if (level < 6)
1179                         ops_run_reconstruct5(sh, percpu, tx);
1180                 else
1181                         ops_run_reconstruct6(sh, percpu, tx);
1182         }
1183
1184         if (test_bit(STRIPE_OP_CHECK, &ops_request)) {
1185                 if (sh->check_state == check_state_run)
1186                         ops_run_check_p(sh, percpu);
1187                 else if (sh->check_state == check_state_run_q)
1188                         ops_run_check_pq(sh, percpu, 0);
1189                 else if (sh->check_state == check_state_run_pq)
1190                         ops_run_check_pq(sh, percpu, 1);
1191                 else
1192                         BUG();
1193         }
1194
1195         if (overlap_clear)
1196                 for (i = disks; i--; ) {
1197                         struct r5dev *dev = &sh->dev[i];
1198                         if (test_and_clear_bit(R5_Overlap, &dev->flags))
1199                                 wake_up(&sh->raid_conf->wait_for_overlap);
1200                 }
1201         put_cpu();
1202 }
1203
1204 static int grow_one_stripe(raid5_conf_t *conf)
1205 {
1206         struct stripe_head *sh;
1207         sh = kmem_cache_alloc(conf->slab_cache, GFP_KERNEL);
1208         if (!sh)
1209                 return 0;
1210         memset(sh, 0, sizeof(*sh) + (conf->raid_disks-1)*sizeof(struct r5dev));
1211         sh->raid_conf = conf;
1212         spin_lock_init(&sh->lock);
1213
1214         if (grow_buffers(sh, conf->raid_disks)) {
1215                 shrink_buffers(sh, conf->raid_disks);
1216                 kmem_cache_free(conf->slab_cache, sh);
1217                 return 0;
1218         }
1219         sh->disks = conf->raid_disks;
1220         /* we just created an active stripe so... */
1221         atomic_set(&sh->count, 1);
1222         atomic_inc(&conf->active_stripes);
1223         INIT_LIST_HEAD(&sh->lru);
1224         release_stripe(sh);
1225         return 1;
1226 }
1227
1228 static int grow_stripes(raid5_conf_t *conf, int num)
1229 {
1230         struct kmem_cache *sc;
1231         int devs = conf->raid_disks;
1232
1233         sprintf(conf->cache_name[0],
1234                 "raid%d-%s", conf->level, mdname(conf->mddev));
1235         sprintf(conf->cache_name[1],
1236                 "raid%d-%s-alt", conf->level, mdname(conf->mddev));
1237         conf->active_name = 0;
1238         sc = kmem_cache_create(conf->cache_name[conf->active_name],
1239                                sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev),
1240                                0, 0, NULL);
1241         if (!sc)
1242                 return 1;
1243         conf->slab_cache = sc;
1244         conf->pool_size = devs;
1245         while (num--)
1246                 if (!grow_one_stripe(conf))
1247                         return 1;
1248         return 0;
1249 }
1250
1251 /**
1252  * scribble_len - return the required size of the scribble region
1253  * @num - total number of disks in the array
1254  *
1255  * The size must be enough to contain:
1256  * 1/ a struct page pointer for each device in the array +2
1257  * 2/ room to convert each entry in (1) to its corresponding dma
1258  *    (dma_map_page()) or page (page_address()) address.
1259  *
1260  * Note: the +2 is for the destination buffers of the ddf/raid6 case where we
1261  * calculate over all devices (not just the data blocks), using zeros in place
1262  * of the P and Q blocks.
1263  */
1264 static size_t scribble_len(int num)
1265 {
1266         size_t len;
1267
1268         len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2);
1269
1270         return len;
1271 }
1272
1273 static int resize_stripes(raid5_conf_t *conf, int newsize)
1274 {
1275         /* Make all the stripes able to hold 'newsize' devices.
1276          * New slots in each stripe get 'page' set to a new page.
1277          *
1278          * This happens in stages:
1279          * 1/ create a new kmem_cache and allocate the required number of
1280          *    stripe_heads.
1281          * 2/ gather all the old stripe_heads and tranfer the pages across
1282          *    to the new stripe_heads.  This will have the side effect of
1283          *    freezing the array as once all stripe_heads have been collected,
1284          *    no IO will be possible.  Old stripe heads are freed once their
1285          *    pages have been transferred over, and the old kmem_cache is
1286          *    freed when all stripes are done.
1287          * 3/ reallocate conf->disks to be suitable bigger.  If this fails,
1288          *    we simple return a failre status - no need to clean anything up.
1289          * 4/ allocate new pages for the new slots in the new stripe_heads.
1290          *    If this fails, we don't bother trying the shrink the
1291          *    stripe_heads down again, we just leave them as they are.
1292          *    As each stripe_head is processed the new one is released into
1293          *    active service.
1294          *
1295          * Once step2 is started, we cannot afford to wait for a write,
1296          * so we use GFP_NOIO allocations.
1297          */
1298         struct stripe_head *osh, *nsh;
1299         LIST_HEAD(newstripes);
1300         struct disk_info *ndisks;
1301         unsigned long cpu;
1302         int err;
1303         struct kmem_cache *sc;
1304         int i;
1305
1306         if (newsize <= conf->pool_size)
1307                 return 0; /* never bother to shrink */
1308
1309         err = md_allow_write(conf->mddev);
1310         if (err)
1311                 return err;
1312
1313         /* Step 1 */
1314         sc = kmem_cache_create(conf->cache_name[1-conf->active_name],
1315                                sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev),
1316                                0, 0, NULL);
1317         if (!sc)
1318                 return -ENOMEM;
1319
1320         for (i = conf->max_nr_stripes; i; i--) {
1321                 nsh = kmem_cache_alloc(sc, GFP_KERNEL);
1322                 if (!nsh)
1323                         break;
1324
1325                 memset(nsh, 0, sizeof(*nsh) + (newsize-1)*sizeof(struct r5dev));
1326
1327                 nsh->raid_conf = conf;
1328                 spin_lock_init(&nsh->lock);
1329
1330                 list_add(&nsh->lru, &newstripes);
1331         }
1332         if (i) {
1333                 /* didn't get enough, give up */
1334                 while (!list_empty(&newstripes)) {
1335                         nsh = list_entry(newstripes.next, struct stripe_head, lru);
1336                         list_del(&nsh->lru);
1337                         kmem_cache_free(sc, nsh);
1338                 }
1339                 kmem_cache_destroy(sc);
1340                 return -ENOMEM;
1341         }
1342         /* Step 2 - Must use GFP_NOIO now.
1343          * OK, we have enough stripes, start collecting inactive
1344          * stripes and copying them over
1345          */
1346         list_for_each_entry(nsh, &newstripes, lru) {
1347                 spin_lock_irq(&conf->device_lock);
1348                 wait_event_lock_irq(conf->wait_for_stripe,
1349                                     !list_empty(&conf->inactive_list),
1350                                     conf->device_lock,
1351                                     unplug_slaves(conf->mddev)
1352                         );
1353                 osh = get_free_stripe(conf);
1354                 spin_unlock_irq(&conf->device_lock);
1355                 atomic_set(&nsh->count, 1);
1356                 for(i=0; i<conf->pool_size; i++)
1357                         nsh->dev[i].page = osh->dev[i].page;
1358                 for( ; i<newsize; i++)
1359                         nsh->dev[i].page = NULL;
1360                 kmem_cache_free(conf->slab_cache, osh);
1361         }
1362         kmem_cache_destroy(conf->slab_cache);
1363
1364         /* Step 3.
1365          * At this point, we are holding all the stripes so the array
1366          * is completely stalled, so now is a good time to resize
1367          * conf->disks and the scribble region
1368          */
1369         ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO);
1370         if (ndisks) {
1371                 for (i=0; i<conf->raid_disks; i++)
1372                         ndisks[i] = conf->disks[i];
1373                 kfree(conf->disks);
1374                 conf->disks = ndisks;
1375         } else
1376                 err = -ENOMEM;
1377
1378         get_online_cpus();
1379         conf->scribble_len = scribble_len(newsize);
1380         for_each_present_cpu(cpu) {
1381                 struct raid5_percpu *percpu;
1382                 void *scribble;
1383
1384                 percpu = per_cpu_ptr(conf->percpu, cpu);
1385                 scribble = kmalloc(conf->scribble_len, GFP_NOIO);
1386
1387                 if (scribble) {
1388                         kfree(percpu->scribble);
1389                         percpu->scribble = scribble;
1390                 } else {
1391                         err = -ENOMEM;
1392                         break;
1393                 }
1394         }
1395         put_online_cpus();
1396
1397         /* Step 4, return new stripes to service */
1398         while(!list_empty(&newstripes)) {
1399                 nsh = list_entry(newstripes.next, struct stripe_head, lru);
1400                 list_del_init(&nsh->lru);
1401
1402                 for (i=conf->raid_disks; i < newsize; i++)
1403                         if (nsh->dev[i].page == NULL) {
1404                                 struct page *p = alloc_page(GFP_NOIO);
1405                                 nsh->dev[i].page = p;
1406                                 if (!p)
1407                                         err = -ENOMEM;
1408                         }
1409                 release_stripe(nsh);
1410         }
1411         /* critical section pass, GFP_NOIO no longer needed */
1412
1413         conf->slab_cache = sc;
1414         conf->active_name = 1-conf->active_name;
1415         conf->pool_size = newsize;
1416         return err;
1417 }
1418
1419 static int drop_one_stripe(raid5_conf_t *conf)
1420 {
1421         struct stripe_head *sh;
1422
1423         spin_lock_irq(&conf->device_lock);
1424         sh = get_free_stripe(conf);
1425         spin_unlock_irq(&conf->device_lock);
1426         if (!sh)
1427                 return 0;
1428         BUG_ON(atomic_read(&sh->count));
1429         shrink_buffers(sh, conf->pool_size);
1430         kmem_cache_free(conf->slab_cache, sh);
1431         atomic_dec(&conf->active_stripes);
1432         return 1;
1433 }
1434
1435 static void shrink_stripes(raid5_conf_t *conf)
1436 {
1437         while (drop_one_stripe(conf))
1438                 ;
1439
1440         if (conf->slab_cache)
1441                 kmem_cache_destroy(conf->slab_cache);
1442         conf->slab_cache = NULL;
1443 }
1444
1445 static void raid5_end_read_request(struct bio * bi, int error)
1446 {
1447         struct stripe_head *sh = bi->bi_private;
1448         raid5_conf_t *conf = sh->raid_conf;
1449         int disks = sh->disks, i;
1450         int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
1451         char b[BDEVNAME_SIZE];
1452         mdk_rdev_t *rdev;
1453
1454
1455         for (i=0 ; i<disks; i++)
1456                 if (bi == &sh->dev[i].req)
1457                         break;
1458
1459         pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n",
1460                 (unsigned long long)sh->sector, i, atomic_read(&sh->count),
1461                 uptodate);
1462         if (i == disks) {
1463                 BUG();
1464                 return;
1465         }
1466
1467         if (uptodate) {
1468                 set_bit(R5_UPTODATE, &sh->dev[i].flags);
1469                 if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
1470                         rdev = conf->disks[i].rdev;
1471                         printk_rl(KERN_INFO "raid5:%s: read error corrected"
1472                                   " (%lu sectors at %llu on %s)\n",
1473                                   mdname(conf->mddev), STRIPE_SECTORS,
1474                                   (unsigned long long)(sh->sector
1475                                                        + rdev->data_offset),
1476                                   bdevname(rdev->bdev, b));
1477                         clear_bit(R5_ReadError, &sh->dev[i].flags);
1478                         clear_bit(R5_ReWrite, &sh->dev[i].flags);
1479                 }
1480                 if (atomic_read(&conf->disks[i].rdev->read_errors))
1481                         atomic_set(&conf->disks[i].rdev->read_errors, 0);
1482         } else {
1483                 const char *bdn = bdevname(conf->disks[i].rdev->bdev, b);
1484                 int retry = 0;
1485                 rdev = conf->disks[i].rdev;
1486
1487                 clear_bit(R5_UPTODATE, &sh->dev[i].flags);
1488                 atomic_inc(&rdev->read_errors);
1489                 if (conf->mddev->degraded)
1490                         printk_rl(KERN_WARNING
1491                                   "raid5:%s: read error not correctable "
1492                                   "(sector %llu on %s).\n",
1493                                   mdname(conf->mddev),
1494                                   (unsigned long long)(sh->sector
1495                                                        + rdev->data_offset),
1496                                   bdn);
1497                 else if (test_bit(R5_ReWrite, &sh->dev[i].flags))
1498                         /* Oh, no!!! */
1499                         printk_rl(KERN_WARNING
1500                                   "raid5:%s: read error NOT corrected!! "
1501                                   "(sector %llu on %s).\n",
1502                                   mdname(conf->mddev),
1503                                   (unsigned long long)(sh->sector
1504                                                        + rdev->data_offset),
1505                                   bdn);
1506                 else if (atomic_read(&rdev->read_errors)
1507                          > conf->max_nr_stripes)
1508                         printk(KERN_WARNING
1509                                "raid5:%s: Too many read errors, failing device %s.\n",
1510                                mdname(conf->mddev), bdn);
1511                 else
1512                         retry = 1;
1513                 if (retry)
1514                         set_bit(R5_ReadError, &sh->dev[i].flags);
1515                 else {
1516                         clear_bit(R5_ReadError, &sh->dev[i].flags);
1517                         clear_bit(R5_ReWrite, &sh->dev[i].flags);
1518                         md_error(conf->mddev, rdev);
1519                 }
1520         }
1521         rdev_dec_pending(conf->disks[i].rdev, conf->mddev);
1522         clear_bit(R5_LOCKED, &sh->dev[i].flags);
1523         set_bit(STRIPE_HANDLE, &sh->state);
1524         release_stripe(sh);
1525 }
1526
1527 static void raid5_end_write_request(struct bio *bi, int error)
1528 {
1529         struct stripe_head *sh = bi->bi_private;
1530         raid5_conf_t *conf = sh->raid_conf;
1531         int disks = sh->disks, i;
1532         int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
1533
1534         for (i=0 ; i<disks; i++)
1535                 if (bi == &sh->dev[i].req)
1536                         break;
1537
1538         pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n",
1539                 (unsigned long long)sh->sector, i, atomic_read(&sh->count),
1540                 uptodate);
1541         if (i == disks) {
1542                 BUG();
1543                 return;
1544         }
1545
1546         if (!uptodate)
1547                 md_error(conf->mddev, conf->disks[i].rdev);
1548
1549         rdev_dec_pending(conf->disks[i].rdev, conf->mddev);
1550
1551         clear_bit(R5_LOCKED, &sh->dev[i].flags);
1552         set_bit(STRIPE_HANDLE, &sh->state);
1553         release_stripe(sh);
1554 }
1555
1556
1557 static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous);
1558
1559 static void raid5_build_block(struct stripe_head *sh, int i, int previous)
1560 {
1561         struct r5dev *dev = &sh->dev[i];
1562
1563         bio_init(&dev->req);
1564         dev->req.bi_io_vec = &dev->vec;
1565         dev->req.bi_vcnt++;
1566         dev->req.bi_max_vecs++;
1567         dev->vec.bv_page = dev->page;
1568         dev->vec.bv_len = STRIPE_SIZE;
1569         dev->vec.bv_offset = 0;
1570
1571         dev->req.bi_sector = sh->sector;
1572         dev->req.bi_private = sh;
1573
1574         dev->flags = 0;
1575         dev->sector = compute_blocknr(sh, i, previous);
1576 }
1577
1578 static void error(mddev_t *mddev, mdk_rdev_t *rdev)
1579 {
1580         char b[BDEVNAME_SIZE];
1581         raid5_conf_t *conf = (raid5_conf_t *) mddev->private;
1582         pr_debug("raid5: error called\n");
1583
1584         if (!test_bit(Faulty, &rdev->flags)) {
1585                 set_bit(MD_CHANGE_DEVS, &mddev->flags);
1586                 if (test_and_clear_bit(In_sync, &rdev->flags)) {
1587                         unsigned long flags;
1588                         spin_lock_irqsave(&conf->device_lock, flags);
1589                         mddev->degraded++;
1590                         spin_unlock_irqrestore(&conf->device_lock, flags);
1591                         /*
1592                          * if recovery was running, make sure it aborts.
1593                          */
1594                         set_bit(MD_RECOVERY_INTR, &mddev->recovery);
1595                 }
1596                 set_bit(Faulty, &rdev->flags);
1597                 printk(KERN_ALERT
1598                        "raid5: Disk failure on %s, disabling device.\n"
1599                        "raid5: Operation continuing on %d devices.\n",
1600                        bdevname(rdev->bdev,b), conf->raid_disks - mddev->degraded);
1601         }
1602 }
1603
1604 /*
1605  * Input: a 'big' sector number,
1606  * Output: index of the data and parity disk, and the sector # in them.
1607  */
1608 static sector_t raid5_compute_sector(raid5_conf_t *conf, sector_t r_sector,
1609                                      int previous, int *dd_idx,
1610                                      struct stripe_head *sh)
1611 {
1612         long stripe;
1613         unsigned long chunk_number;
1614         unsigned int chunk_offset;
1615         int pd_idx, qd_idx;
1616         int ddf_layout = 0;
1617         sector_t new_sector;
1618         int algorithm = previous ? conf->prev_algo
1619                                  : conf->algorithm;
1620         int sectors_per_chunk = previous ? conf->prev_chunk_sectors
1621                                          : conf->chunk_sectors;
1622         int raid_disks = previous ? conf->previous_raid_disks
1623                                   : conf->raid_disks;
1624         int data_disks = raid_disks - conf->max_degraded;
1625
1626         /* First compute the information on this sector */
1627
1628         /*
1629          * Compute the chunk number and the sector offset inside the chunk
1630          */
1631         chunk_offset = sector_div(r_sector, sectors_per_chunk);
1632         chunk_number = r_sector;
1633         BUG_ON(r_sector != chunk_number);
1634
1635         /*
1636          * Compute the stripe number
1637          */
1638         stripe = chunk_number / data_disks;
1639
1640         /*
1641          * Compute the data disk and parity disk indexes inside the stripe
1642          */
1643         *dd_idx = chunk_number % data_disks;
1644
1645         /*
1646          * Select the parity disk based on the user selected algorithm.
1647          */
1648         pd_idx = qd_idx = ~0;
1649         switch(conf->level) {
1650         case 4:
1651                 pd_idx = data_disks;
1652                 break;
1653         case 5:
1654                 switch (algorithm) {
1655                 case ALGORITHM_LEFT_ASYMMETRIC:
1656                         pd_idx = data_disks - stripe % raid_disks;
1657                         if (*dd_idx >= pd_idx)
1658                                 (*dd_idx)++;
1659                         break;
1660                 case ALGORITHM_RIGHT_ASYMMETRIC:
1661                         pd_idx = stripe % raid_disks;
1662                         if (*dd_idx >= pd_idx)
1663                                 (*dd_idx)++;
1664                         break;
1665                 case ALGORITHM_LEFT_SYMMETRIC:
1666                         pd_idx = data_disks - stripe % raid_disks;
1667                         *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
1668                         break;
1669                 case ALGORITHM_RIGHT_SYMMETRIC:
1670                         pd_idx = stripe % raid_disks;
1671                         *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
1672                         break;
1673                 case ALGORITHM_PARITY_0:
1674                         pd_idx = 0;
1675                         (*dd_idx)++;
1676                         break;
1677                 case ALGORITHM_PARITY_N:
1678                         pd_idx = data_disks;
1679                         break;
1680                 default:
1681                         printk(KERN_ERR "raid5: unsupported algorithm %d\n",
1682                                 algorithm);
1683                         BUG();
1684                 }
1685                 break;
1686         case 6:
1687
1688                 switch (algorithm) {
1689                 case ALGORITHM_LEFT_ASYMMETRIC:
1690                         pd_idx = raid_disks - 1 - (stripe % raid_disks);
1691                         qd_idx = pd_idx + 1;
1692                         if (pd_idx == raid_disks-1) {
1693                                 (*dd_idx)++;    /* Q D D D P */
1694                                 qd_idx = 0;
1695                         } else if (*dd_idx >= pd_idx)
1696                                 (*dd_idx) += 2; /* D D P Q D */
1697                         break;
1698                 case ALGORITHM_RIGHT_ASYMMETRIC:
1699                         pd_idx = stripe % raid_disks;
1700                         qd_idx = pd_idx + 1;
1701                         if (pd_idx == raid_disks-1) {
1702                                 (*dd_idx)++;    /* Q D D D P */
1703                                 qd_idx = 0;
1704                         } else if (*dd_idx >= pd_idx)
1705                                 (*dd_idx) += 2; /* D D P Q D */
1706                         break;
1707                 case ALGORITHM_LEFT_SYMMETRIC:
1708                         pd_idx = raid_disks - 1 - (stripe % raid_disks);
1709                         qd_idx = (pd_idx + 1) % raid_disks;
1710                         *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
1711                         break;
1712                 case ALGORITHM_RIGHT_SYMMETRIC:
1713                         pd_idx = stripe % raid_disks;
1714                         qd_idx = (pd_idx + 1) % raid_disks;
1715                         *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
1716                         break;
1717
1718                 case ALGORITHM_PARITY_0:
1719                         pd_idx = 0;
1720                         qd_idx = 1;
1721                         (*dd_idx) += 2;
1722                         break;
1723                 case ALGORITHM_PARITY_N:
1724                         pd_idx = data_disks;
1725                         qd_idx = data_disks + 1;
1726                         break;
1727
1728                 case ALGORITHM_ROTATING_ZERO_RESTART:
1729                         /* Exactly the same as RIGHT_ASYMMETRIC, but or
1730                          * of blocks for computing Q is different.
1731                          */
1732                         pd_idx = stripe % raid_disks;
1733                         qd_idx = pd_idx + 1;
1734                         if (pd_idx == raid_disks-1) {
1735                                 (*dd_idx)++;    /* Q D D D P */
1736                                 qd_idx = 0;
1737                         } else if (*dd_idx >= pd_idx)
1738                                 (*dd_idx) += 2; /* D D P Q D */
1739                         ddf_layout = 1;
1740                         break;
1741
1742                 case ALGORITHM_ROTATING_N_RESTART:
1743                         /* Same a left_asymmetric, by first stripe is
1744                          * D D D P Q  rather than
1745                          * Q D D D P
1746                          */
1747                         pd_idx = raid_disks - 1 - ((stripe + 1) % raid_disks);
1748                         qd_idx = pd_idx + 1;
1749                         if (pd_idx == raid_disks-1) {
1750                                 (*dd_idx)++;    /* Q D D D P */
1751                                 qd_idx = 0;
1752                         } else if (*dd_idx >= pd_idx)
1753                                 (*dd_idx) += 2; /* D D P Q D */
1754                         ddf_layout = 1;
1755                         break;
1756
1757                 case ALGORITHM_ROTATING_N_CONTINUE:
1758                         /* Same as left_symmetric but Q is before P */
1759                         pd_idx = raid_disks - 1 - (stripe % raid_disks);
1760                         qd_idx = (pd_idx + raid_disks - 1) % raid_disks;
1761                         *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
1762                         ddf_layout = 1;
1763                         break;
1764
1765                 case ALGORITHM_LEFT_ASYMMETRIC_6:
1766                         /* RAID5 left_asymmetric, with Q on last device */
1767                         pd_idx = data_disks - stripe % (raid_disks-1);
1768                         if (*dd_idx >= pd_idx)
1769                                 (*dd_idx)++;
1770                         qd_idx = raid_disks - 1;
1771                         break;
1772
1773                 case ALGORITHM_RIGHT_ASYMMETRIC_6:
1774                         pd_idx = stripe % (raid_disks-1);
1775                         if (*dd_idx >= pd_idx)
1776                                 (*dd_idx)++;
1777                         qd_idx = raid_disks - 1;
1778                         break;
1779
1780                 case ALGORITHM_LEFT_SYMMETRIC_6:
1781                         pd_idx = data_disks - stripe % (raid_disks-1);
1782                         *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
1783                         qd_idx = raid_disks - 1;
1784                         break;
1785
1786                 case ALGORITHM_RIGHT_SYMMETRIC_6:
1787                         pd_idx = stripe % (raid_disks-1);
1788                         *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
1789                         qd_idx = raid_disks - 1;
1790                         break;
1791
1792                 case ALGORITHM_PARITY_0_6:
1793                         pd_idx = 0;
1794                         (*dd_idx)++;
1795                         qd_idx = raid_disks - 1;
1796                         break;
1797
1798
1799                 default:
1800                         printk(KERN_CRIT "raid6: unsupported algorithm %d\n",
1801                                algorithm);
1802                         BUG();
1803                 }
1804                 break;
1805         }
1806
1807         if (sh) {
1808                 sh->pd_idx = pd_idx;
1809                 sh->qd_idx = qd_idx;
1810                 sh->ddf_layout = ddf_layout;
1811         }
1812         /*
1813          * Finally, compute the new sector number
1814          */
1815         new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset;
1816         return new_sector;
1817 }
1818
1819
1820 static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous)
1821 {
1822         raid5_conf_t *conf = sh->raid_conf;
1823         int raid_disks = sh->disks;
1824         int data_disks = raid_disks - conf->max_degraded;
1825         sector_t new_sector = sh->sector, check;
1826         int sectors_per_chunk = previous ? conf->prev_chunk_sectors
1827                                          : conf->chunk_sectors;
1828         int algorithm = previous ? conf->prev_algo
1829                                  : conf->algorithm;
1830         sector_t stripe;
1831         int chunk_offset;
1832         int chunk_number, dummy1, dd_idx = i;
1833         sector_t r_sector;
1834         struct stripe_head sh2;
1835
1836
1837         chunk_offset = sector_div(new_sector, sectors_per_chunk);
1838         stripe = new_sector;
1839         BUG_ON(new_sector != stripe);
1840
1841         if (i == sh->pd_idx)
1842                 return 0;
1843         switch(conf->level) {
1844         case 4: break;
1845         case 5:
1846                 switch (algorithm) {
1847                 case ALGORITHM_LEFT_ASYMMETRIC:
1848                 case ALGORITHM_RIGHT_ASYMMETRIC:
1849                         if (i > sh->pd_idx)
1850                                 i--;
1851                         break;
1852                 case ALGORITHM_LEFT_SYMMETRIC:
1853                 case ALGORITHM_RIGHT_SYMMETRIC:
1854                         if (i < sh->pd_idx)
1855                                 i += raid_disks;
1856                         i -= (sh->pd_idx + 1);
1857                         break;
1858                 case ALGORITHM_PARITY_0:
1859                         i -= 1;
1860                         break;
1861                 case ALGORITHM_PARITY_N:
1862                         break;
1863                 default:
1864                         printk(KERN_ERR "raid5: unsupported algorithm %d\n",
1865                                algorithm);
1866                         BUG();
1867                 }
1868                 break;
1869         case 6:
1870                 if (i == sh->qd_idx)
1871                         return 0; /* It is the Q disk */
1872                 switch (algorithm) {
1873                 case ALGORITHM_LEFT_ASYMMETRIC:
1874                 case ALGORITHM_RIGHT_ASYMMETRIC:
1875                 case ALGORITHM_ROTATING_ZERO_RESTART:
1876                 case ALGORITHM_ROTATING_N_RESTART:
1877                         if (sh->pd_idx == raid_disks-1)
1878                                 i--;    /* Q D D D P */
1879                         else if (i > sh->pd_idx)
1880                                 i -= 2; /* D D P Q D */
1881                         break;
1882                 case ALGORITHM_LEFT_SYMMETRIC:
1883                 case ALGORITHM_RIGHT_SYMMETRIC:
1884                         if (sh->pd_idx == raid_disks-1)
1885                                 i--; /* Q D D D P */
1886                         else {
1887                                 /* D D P Q D */
1888                                 if (i < sh->pd_idx)
1889                                         i += raid_disks;
1890                                 i -= (sh->pd_idx + 2);
1891                         }
1892                         break;
1893                 case ALGORITHM_PARITY_0:
1894                         i -= 2;
1895                         break;
1896                 case ALGORITHM_PARITY_N:
1897                         break;
1898                 case ALGORITHM_ROTATING_N_CONTINUE:
1899                         if (sh->pd_idx == 0)
1900                                 i--;    /* P D D D Q */
1901                         else if (i > sh->pd_idx)
1902                                 i -= 2; /* D D Q P D */
1903                         break;
1904                 case ALGORITHM_LEFT_ASYMMETRIC_6:
1905                 case ALGORITHM_RIGHT_ASYMMETRIC_6:
1906                         if (i > sh->pd_idx)
1907                                 i--;
1908                         break;
1909                 case ALGORITHM_LEFT_SYMMETRIC_6:
1910                 case ALGORITHM_RIGHT_SYMMETRIC_6:
1911                         if (i < sh->pd_idx)
1912                                 i += data_disks + 1;
1913                         i -= (sh->pd_idx + 1);
1914                         break;
1915                 case ALGORITHM_PARITY_0_6:
1916                         i -= 1;
1917                         break;
1918                 default:
1919                         printk(KERN_CRIT "raid6: unsupported algorithm %d\n",
1920                                algorithm);
1921                         BUG();
1922                 }
1923                 break;
1924         }
1925
1926         chunk_number = stripe * data_disks + i;
1927         r_sector = (sector_t)chunk_number * sectors_per_chunk + chunk_offset;
1928
1929         check = raid5_compute_sector(conf, r_sector,
1930                                      previous, &dummy1, &sh2);
1931         if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx
1932                 || sh2.qd_idx != sh->qd_idx) {
1933                 printk(KERN_ERR "compute_blocknr: map not correct\n");
1934                 return 0;
1935         }
1936         return r_sector;
1937 }
1938
1939
1940 static void
1941 schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s,
1942                          int rcw, int expand)
1943 {
1944         int i, pd_idx = sh->pd_idx, disks = sh->disks;
1945         raid5_conf_t *conf = sh->raid_conf;
1946         int level = conf->level;
1947
1948         if (rcw) {
1949                 /* if we are not expanding this is a proper write request, and
1950                  * there will be bios with new data to be drained into the
1951                  * stripe cache
1952                  */
1953                 if (!expand) {
1954                         sh->reconstruct_state = reconstruct_state_drain_run;
1955                         set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
1956                 } else
1957                         sh->reconstruct_state = reconstruct_state_run;
1958
1959                 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
1960
1961                 for (i = disks; i--; ) {
1962                         struct r5dev *dev = &sh->dev[i];
1963
1964                         if (dev->towrite) {
1965                                 set_bit(R5_LOCKED, &dev->flags);
1966                                 set_bit(R5_Wantdrain, &dev->flags);
1967                                 if (!expand)
1968                                         clear_bit(R5_UPTODATE, &dev->flags);
1969                                 s->locked++;
1970                         }
1971                 }
1972                 if (s->locked + conf->max_degraded == disks)
1973                         if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state))
1974                                 atomic_inc(&conf->pending_full_writes);
1975         } else {
1976                 BUG_ON(level == 6);
1977                 BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) ||
1978                         test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags)));
1979
1980                 sh->reconstruct_state = reconstruct_state_prexor_drain_run;
1981                 set_bit(STRIPE_OP_PREXOR, &s->ops_request);
1982                 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
1983                 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
1984
1985                 for (i = disks; i--; ) {
1986                         struct r5dev *dev = &sh->dev[i];
1987                         if (i == pd_idx)
1988                                 continue;
1989
1990                         if (dev->towrite &&
1991                             (test_bit(R5_UPTODATE, &dev->flags) ||
1992                              test_bit(R5_Wantcompute, &dev->flags))) {
1993                                 set_bit(R5_Wantdrain, &dev->flags);
1994                                 set_bit(R5_LOCKED, &dev->flags);
1995                                 clear_bit(R5_UPTODATE, &dev->flags);
1996                                 s->locked++;
1997                         }
1998                 }
1999         }
2000
2001         /* keep the parity disk(s) locked while asynchronous operations
2002          * are in flight
2003          */
2004         set_bit(R5_LOCKED, &sh->dev[pd_idx].flags);
2005         clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
2006         s->locked++;
2007
2008         if (level == 6) {
2009                 int qd_idx = sh->qd_idx;
2010                 struct r5dev *dev = &sh->dev[qd_idx];
2011
2012                 set_bit(R5_LOCKED, &dev->flags);
2013                 clear_bit(R5_UPTODATE, &dev->flags);
2014                 s->locked++;
2015         }
2016
2017         pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n",
2018                 __func__, (unsigned long long)sh->sector,
2019                 s->locked, s->ops_request);
2020 }
2021
2022 /*
2023  * Each stripe/dev can have one or more bion attached.
2024  * toread/towrite point to the first in a chain.
2025  * The bi_next chain must be in order.
2026  */
2027 static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx, int forwrite)
2028 {
2029         struct bio **bip;
2030         raid5_conf_t *conf = sh->raid_conf;
2031         int firstwrite=0;
2032
2033         pr_debug("adding bh b#%llu to stripe s#%llu\n",
2034                 (unsigned long long)bi->bi_sector,
2035                 (unsigned long long)sh->sector);
2036
2037
2038         spin_lock(&sh->lock);
2039         spin_lock_irq(&conf->device_lock);
2040         if (forwrite) {
2041                 bip = &sh->dev[dd_idx].towrite;
2042                 if (*bip == NULL && sh->dev[dd_idx].written == NULL)
2043                         firstwrite = 1;
2044         } else
2045                 bip = &sh->dev[dd_idx].toread;
2046         while (*bip && (*bip)->bi_sector < bi->bi_sector) {
2047                 if ((*bip)->bi_sector + ((*bip)->bi_size >> 9) > bi->bi_sector)
2048                         goto overlap;
2049                 bip = & (*bip)->bi_next;
2050         }
2051         if (*bip && (*bip)->bi_sector < bi->bi_sector + ((bi->bi_size)>>9))
2052                 goto overlap;
2053
2054         BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next);
2055         if (*bip)
2056                 bi->bi_next = *bip;
2057         *bip = bi;
2058         bi->bi_phys_segments++;
2059         spin_unlock_irq(&conf->device_lock);
2060         spin_unlock(&sh->lock);
2061
2062         pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n",
2063                 (unsigned long long)bi->bi_sector,
2064                 (unsigned long long)sh->sector, dd_idx);
2065
2066         if (conf->mddev->bitmap && firstwrite) {
2067                 bitmap_startwrite(conf->mddev->bitmap, sh->sector,
2068                                   STRIPE_SECTORS, 0);
2069                 sh->bm_seq = conf->seq_flush+1;
2070                 set_bit(STRIPE_BIT_DELAY, &sh->state);
2071         }
2072
2073         if (forwrite) {
2074                 /* check if page is covered */
2075                 sector_t sector = sh->dev[dd_idx].sector;
2076                 for (bi=sh->dev[dd_idx].towrite;
2077                      sector < sh->dev[dd_idx].sector + STRIPE_SECTORS &&
2078                              bi && bi->bi_sector <= sector;
2079                      bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) {
2080                         if (bi->bi_sector + (bi->bi_size>>9) >= sector)
2081                                 sector = bi->bi_sector + (bi->bi_size>>9);
2082                 }
2083                 if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS)
2084                         set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags);
2085         }
2086         return 1;
2087
2088  overlap:
2089         set_bit(R5_Overlap, &sh->dev[dd_idx].flags);
2090         spin_unlock_irq(&conf->device_lock);
2091         spin_unlock(&sh->lock);
2092         return 0;
2093 }
2094
2095 static void end_reshape(raid5_conf_t *conf);
2096
2097 static void stripe_set_idx(sector_t stripe, raid5_conf_t *conf, int previous,
2098                             struct stripe_head *sh)
2099 {
2100         int sectors_per_chunk =
2101                 previous ? conf->prev_chunk_sectors : conf->chunk_sectors;
2102         int dd_idx;
2103         int chunk_offset = sector_div(stripe, sectors_per_chunk);
2104         int disks = previous ? conf->previous_raid_disks : conf->raid_disks;
2105
2106         raid5_compute_sector(conf,
2107                              stripe * (disks - conf->max_degraded)
2108                              *sectors_per_chunk + chunk_offset,
2109                              previous,
2110                              &dd_idx, sh);
2111 }
2112
2113 static void
2114 handle_failed_stripe(raid5_conf_t *conf, struct stripe_head *sh,
2115                                 struct stripe_head_state *s, int disks,
2116                                 struct bio **return_bi)
2117 {
2118         int i;
2119         for (i = disks; i--; ) {
2120                 struct bio *bi;
2121                 int bitmap_end = 0;
2122
2123                 if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
2124                         mdk_rdev_t *rdev;
2125                         rcu_read_lock();
2126                         rdev = rcu_dereference(conf->disks[i].rdev);
2127                         if (rdev && test_bit(In_sync, &rdev->flags))
2128                                 /* multiple read failures in one stripe */
2129                                 md_error(conf->mddev, rdev);
2130                         rcu_read_unlock();
2131                 }
2132                 spin_lock_irq(&conf->device_lock);
2133                 /* fail all writes first */
2134                 bi = sh->dev[i].towrite;
2135                 sh->dev[i].towrite = NULL;
2136                 if (bi) {
2137                         s->to_write--;
2138                         bitmap_end = 1;
2139                 }
2140
2141                 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
2142                         wake_up(&conf->wait_for_overlap);
2143
2144                 while (bi && bi->bi_sector <
2145                         sh->dev[i].sector + STRIPE_SECTORS) {
2146                         struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector);
2147                         clear_bit(BIO_UPTODATE, &bi->bi_flags);
2148                         if (!raid5_dec_bi_phys_segments(bi)) {
2149                                 md_write_end(conf->mddev);
2150                                 bi->bi_next = *return_bi;
2151                                 *return_bi = bi;
2152                         }
2153                         bi = nextbi;
2154                 }
2155                 /* and fail all 'written' */
2156                 bi = sh->dev[i].written;
2157                 sh->dev[i].written = NULL;
2158                 if (bi) bitmap_end = 1;
2159                 while (bi && bi->bi_sector <
2160                        sh->dev[i].sector + STRIPE_SECTORS) {
2161                         struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector);
2162                         clear_bit(BIO_UPTODATE, &bi->bi_flags);
2163                         if (!raid5_dec_bi_phys_segments(bi)) {
2164                                 md_write_end(conf->mddev);
2165                                 bi->bi_next = *return_bi;
2166                                 *return_bi = bi;
2167                         }
2168                         bi = bi2;
2169                 }
2170
2171                 /* fail any reads if this device is non-operational and
2172                  * the data has not reached the cache yet.
2173                  */
2174                 if (!test_bit(R5_Wantfill, &sh->dev[i].flags) &&
2175                     (!test_bit(R5_Insync, &sh->dev[i].flags) ||
2176                       test_bit(R5_ReadError, &sh->dev[i].flags))) {
2177                         bi = sh->dev[i].toread;
2178                         sh->dev[i].toread = NULL;
2179                         if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
2180                                 wake_up(&conf->wait_for_overlap);
2181                         if (bi) s->to_read--;
2182                         while (bi && bi->bi_sector <
2183                                sh->dev[i].sector + STRIPE_SECTORS) {
2184                                 struct bio *nextbi =
2185                                         r5_next_bio(bi, sh->dev[i].sector);
2186                                 clear_bit(BIO_UPTODATE, &bi->bi_flags);
2187                                 if (!raid5_dec_bi_phys_segments(bi)) {
2188                                         bi->bi_next = *return_bi;
2189                                         *return_bi = bi;
2190                                 }
2191                                 bi = nextbi;
2192                         }
2193                 }
2194                 spin_unlock_irq(&conf->device_lock);
2195                 if (bitmap_end)
2196                         bitmap_endwrite(conf->mddev->bitmap, sh->sector,
2197                                         STRIPE_SECTORS, 0, 0);
2198         }
2199
2200         if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
2201                 if (atomic_dec_and_test(&conf->pending_full_writes))
2202                         md_wakeup_thread(conf->mddev->thread);
2203 }
2204
2205 /* fetch_block5 - checks the given member device to see if its data needs
2206  * to be read or computed to satisfy a request.
2207  *
2208  * Returns 1 when no more member devices need to be checked, otherwise returns
2209  * 0 to tell the loop in handle_stripe_fill5 to continue
2210  */
2211 static int fetch_block5(struct stripe_head *sh, struct stripe_head_state *s,
2212                         int disk_idx, int disks)
2213 {
2214         struct r5dev *dev = &sh->dev[disk_idx];
2215         struct r5dev *failed_dev = &sh->dev[s->failed_num];
2216
2217         /* is the data in this block needed, and can we get it? */
2218         if (!test_bit(R5_LOCKED, &dev->flags) &&
2219             !test_bit(R5_UPTODATE, &dev->flags) &&
2220             (dev->toread ||
2221              (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)) ||
2222              s->syncing || s->expanding ||
2223              (s->failed &&
2224               (failed_dev->toread ||
2225                (failed_dev->towrite &&
2226                 !test_bit(R5_OVERWRITE, &failed_dev->flags)))))) {
2227                 /* We would like to get this block, possibly by computing it,
2228                  * otherwise read it if the backing disk is insync
2229                  */
2230                 if ((s->uptodate == disks - 1) &&
2231                     (s->failed && disk_idx == s->failed_num)) {
2232                         set_bit(STRIPE_COMPUTE_RUN, &sh->state);
2233                         set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
2234                         set_bit(R5_Wantcompute, &dev->flags);
2235                         sh->ops.target = disk_idx;
2236                         sh->ops.target2 = -1;
2237                         s->req_compute = 1;
2238                         /* Careful: from this point on 'uptodate' is in the eye
2239                          * of raid_run_ops which services 'compute' operations
2240                          * before writes. R5_Wantcompute flags a block that will
2241                          * be R5_UPTODATE by the time it is needed for a
2242                          * subsequent operation.
2243                          */
2244                         s->uptodate++;
2245                         return 1; /* uptodate + compute == disks */
2246                 } else if (test_bit(R5_Insync, &dev->flags)) {
2247                         set_bit(R5_LOCKED, &dev->flags);
2248                         set_bit(R5_Wantread, &dev->flags);
2249                         s->locked++;
2250                         pr_debug("Reading block %d (sync=%d)\n", disk_idx,
2251                                 s->syncing);
2252                 }
2253         }
2254
2255         return 0;
2256 }
2257
2258 /**
2259  * handle_stripe_fill5 - read or compute data to satisfy pending requests.
2260  */
2261 static void handle_stripe_fill5(struct stripe_head *sh,
2262                         struct stripe_head_state *s, int disks)
2263 {
2264         int i;
2265
2266         /* look for blocks to read/compute, skip this if a compute
2267          * is already in flight, or if the stripe contents are in the
2268          * midst of changing due to a write
2269          */
2270         if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state &&
2271             !sh->reconstruct_state)
2272                 for (i = disks; i--; )
2273                         if (fetch_block5(sh, s, i, disks))
2274                                 break;
2275         set_bit(STRIPE_HANDLE, &sh->state);
2276 }
2277
2278 /* fetch_block6 - checks the given member device to see if its data needs
2279  * to be read or computed to satisfy a request.
2280  *
2281  * Returns 1 when no more member devices need to be checked, otherwise returns
2282  * 0 to tell the loop in handle_stripe_fill6 to continue
2283  */
2284 static int fetch_block6(struct stripe_head *sh, struct stripe_head_state *s,
2285                          struct r6_state *r6s, int disk_idx, int disks)
2286 {
2287         struct r5dev *dev = &sh->dev[disk_idx];
2288         struct r5dev *fdev[2] = { &sh->dev[r6s->failed_num[0]],
2289                                   &sh->dev[r6s->failed_num[1]] };
2290
2291         if (!test_bit(R5_LOCKED, &dev->flags) &&
2292             !test_bit(R5_UPTODATE, &dev->flags) &&
2293             (dev->toread ||
2294              (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)) ||
2295              s->syncing || s->expanding ||
2296              (s->failed >= 1 &&
2297               (fdev[0]->toread || s->to_write)) ||
2298              (s->failed >= 2 &&
2299               (fdev[1]->toread || s->to_write)))) {
2300                 /* we would like to get this block, possibly by computing it,
2301                  * otherwise read it if the backing disk is insync
2302                  */
2303                 BUG_ON(test_bit(R5_Wantcompute, &dev->flags));
2304                 BUG_ON(test_bit(R5_Wantread, &dev->flags));
2305                 if ((s->uptodate == disks - 1) &&
2306                     (s->failed && (disk_idx == r6s->failed_num[0] ||
2307                                    disk_idx == r6s->failed_num[1]))) {
2308                         /* have disk failed, and we're requested to fetch it;
2309                          * do compute it
2310                          */
2311                         pr_debug("Computing stripe %llu block %d\n",
2312                                (unsigned long long)sh->sector, disk_idx);
2313                         set_bit(STRIPE_COMPUTE_RUN, &sh->state);
2314                         set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
2315                         set_bit(R5_Wantcompute, &dev->flags);
2316                         sh->ops.target = disk_idx;
2317                         sh->ops.target2 = -1; /* no 2nd target */
2318                         s->req_compute = 1;
2319                         s->uptodate++;
2320                         return 1;
2321                 } else if (s->uptodate == disks-2 && s->failed >= 2) {
2322                         /* Computing 2-failure is *very* expensive; only
2323                          * do it if failed >= 2
2324                          */
2325                         int other;
2326                         for (other = disks; other--; ) {
2327                                 if (other == disk_idx)
2328                                         continue;
2329                                 if (!test_bit(R5_UPTODATE,
2330                                       &sh->dev[other].flags))
2331                                         break;
2332                         }
2333                         BUG_ON(other < 0);
2334                         pr_debug("Computing stripe %llu blocks %d,%d\n",
2335                                (unsigned long long)sh->sector,
2336                                disk_idx, other);
2337                         set_bit(STRIPE_COMPUTE_RUN, &sh->state);
2338                         set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
2339                         set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags);
2340                         set_bit(R5_Wantcompute, &sh->dev[other].flags);
2341                         sh->ops.target = disk_idx;
2342                         sh->ops.target2 = other;
2343                         s->uptodate += 2;
2344                         s->req_compute = 1;
2345                         return 1;
2346                 } else if (test_bit(R5_Insync, &dev->flags)) {
2347                         set_bit(R5_LOCKED, &dev->flags);
2348                         set_bit(R5_Wantread, &dev->flags);
2349                         s->locked++;
2350                         pr_debug("Reading block %d (sync=%d)\n",
2351                                 disk_idx, s->syncing);
2352                 }
2353         }
2354
2355         return 0;
2356 }
2357
2358 /**
2359  * handle_stripe_fill6 - read or compute data to satisfy pending requests.
2360  */
2361 static void handle_stripe_fill6(struct stripe_head *sh,
2362                         struct stripe_head_state *s, struct r6_state *r6s,
2363                         int disks)
2364 {
2365         int i;
2366
2367         /* look for blocks to read/compute, skip this if a compute
2368          * is already in flight, or if the stripe contents are in the
2369          * midst of changing due to a write
2370          */
2371         if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state &&
2372             !sh->reconstruct_state)
2373                 for (i = disks; i--; )
2374                         if (fetch_block6(sh, s, r6s, i, disks))
2375                                 break;
2376         set_bit(STRIPE_HANDLE, &sh->state);
2377 }
2378
2379
2380 /* handle_stripe_clean_event
2381  * any written block on an uptodate or failed drive can be returned.
2382  * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but
2383  * never LOCKED, so we don't need to test 'failed' directly.
2384  */
2385 static void handle_stripe_clean_event(raid5_conf_t *conf,
2386         struct stripe_head *sh, int disks, struct bio **return_bi)
2387 {
2388         int i;
2389         struct r5dev *dev;
2390
2391         for (i = disks; i--; )
2392                 if (sh->dev[i].written) {
2393                         dev = &sh->dev[i];
2394                         if (!test_bit(R5_LOCKED, &dev->flags) &&
2395                                 test_bit(R5_UPTODATE, &dev->flags)) {
2396                                 /* We can return any write requests */
2397                                 struct bio *wbi, *wbi2;
2398                                 int bitmap_end = 0;
2399                                 pr_debug("Return write for disc %d\n", i);
2400                                 spin_lock_irq(&conf->device_lock);
2401                                 wbi = dev->written;
2402                                 dev->written = NULL;
2403                                 while (wbi && wbi->bi_sector <
2404                                         dev->sector + STRIPE_SECTORS) {
2405                                         wbi2 = r5_next_bio(wbi, dev->sector);
2406                                         if (!raid5_dec_bi_phys_segments(wbi)) {
2407                                                 md_write_end(conf->mddev);
2408                                                 wbi->bi_next = *return_bi;
2409                                                 *return_bi = wbi;
2410                                         }
2411                                         wbi = wbi2;
2412                                 }
2413                                 if (dev->towrite == NULL)
2414                                         bitmap_end = 1;
2415                                 spin_unlock_irq(&conf->device_lock);
2416                                 if (bitmap_end)
2417                                         bitmap_endwrite(conf->mddev->bitmap,
2418                                                         sh->sector,
2419                                                         STRIPE_SECTORS,
2420                                          !test_bit(STRIPE_DEGRADED, &sh->state),
2421                                                         0);
2422                         }
2423                 }
2424
2425         if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
2426                 if (atomic_dec_and_test(&conf->pending_full_writes))
2427                         md_wakeup_thread(conf->mddev->thread);
2428 }
2429
2430 static void handle_stripe_dirtying5(raid5_conf_t *conf,
2431                 struct stripe_head *sh, struct stripe_head_state *s, int disks)
2432 {
2433         int rmw = 0, rcw = 0, i;
2434         for (i = disks; i--; ) {
2435                 /* would I have to read this buffer for read_modify_write */
2436                 struct r5dev *dev = &sh->dev[i];
2437                 if ((dev->towrite || i == sh->pd_idx) &&
2438                     !test_bit(R5_LOCKED, &dev->flags) &&
2439                     !(test_bit(R5_UPTODATE, &dev->flags) ||
2440                       test_bit(R5_Wantcompute, &dev->flags))) {
2441                         if (test_bit(R5_Insync, &dev->flags))
2442                                 rmw++;
2443                         else
2444                                 rmw += 2*disks;  /* cannot read it */
2445                 }
2446                 /* Would I have to read this buffer for reconstruct_write */
2447                 if (!test_bit(R5_OVERWRITE, &dev->flags) && i != sh->pd_idx &&
2448                     !test_bit(R5_LOCKED, &dev->flags) &&
2449                     !(test_bit(R5_UPTODATE, &dev->flags) ||
2450                     test_bit(R5_Wantcompute, &dev->flags))) {
2451                         if (test_bit(R5_Insync, &dev->flags)) rcw++;
2452                         else
2453                                 rcw += 2*disks;
2454                 }
2455         }
2456         pr_debug("for sector %llu, rmw=%d rcw=%d\n",
2457                 (unsigned long long)sh->sector, rmw, rcw);
2458         set_bit(STRIPE_HANDLE, &sh->state);
2459         if (rmw < rcw && rmw > 0)
2460                 /* prefer read-modify-write, but need to get some data */
2461                 for (i = disks; i--; ) {
2462                         struct r5dev *dev = &sh->dev[i];
2463                         if ((dev->towrite || i == sh->pd_idx) &&
2464                             !test_bit(R5_LOCKED, &dev->flags) &&
2465                             !(test_bit(R5_UPTODATE, &dev->flags) ||
2466                             test_bit(R5_Wantcompute, &dev->flags)) &&
2467                             test_bit(R5_Insync, &dev->flags)) {
2468                                 if (
2469                                   test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
2470                                         pr_debug("Read_old block "
2471                                                 "%d for r-m-w\n", i);
2472                                         set_bit(R5_LOCKED, &dev->flags);
2473                                         set_bit(R5_Wantread, &dev->flags);
2474                                         s->locked++;
2475                                 } else {
2476                                         set_bit(STRIPE_DELAYED, &sh->state);
2477                                         set_bit(STRIPE_HANDLE, &sh->state);
2478                                 }
2479                         }
2480                 }
2481         if (rcw <= rmw && rcw > 0)
2482                 /* want reconstruct write, but need to get some data */
2483                 for (i = disks; i--; ) {
2484                         struct r5dev *dev = &sh->dev[i];
2485                         if (!test_bit(R5_OVERWRITE, &dev->flags) &&
2486                             i != sh->pd_idx &&
2487                             !test_bit(R5_LOCKED, &dev->flags) &&
2488                             !(test_bit(R5_UPTODATE, &dev->flags) ||
2489                             test_bit(R5_Wantcompute, &dev->flags)) &&
2490                             test_bit(R5_Insync, &dev->flags)) {
2491                                 if (
2492                                   test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
2493                                         pr_debug("Read_old block "
2494                                                 "%d for Reconstruct\n", i);
2495                                         set_bit(R5_LOCKED, &dev->flags);
2496                                         set_bit(R5_Wantread, &dev->flags);
2497                                         s->locked++;
2498                                 } else {
2499                                         set_bit(STRIPE_DELAYED, &sh->state);
2500                                         set_bit(STRIPE_HANDLE, &sh->state);
2501                                 }
2502                         }
2503                 }
2504         /* now if nothing is locked, and if we have enough data,
2505          * we can start a write request
2506          */
2507         /* since handle_stripe can be called at any time we need to handle the
2508          * case where a compute block operation has been submitted and then a
2509          * subsequent call wants to start a write request.  raid_run_ops only
2510          * handles the case where compute block and reconstruct are requested
2511          * simultaneously.  If this is not the case then new writes need to be
2512          * held off until the compute completes.
2513          */
2514         if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&
2515             (s->locked == 0 && (rcw == 0 || rmw == 0) &&
2516             !test_bit(STRIPE_BIT_DELAY, &sh->state)))
2517                 schedule_reconstruction(sh, s, rcw == 0, 0);
2518 }
2519
2520 static void handle_stripe_dirtying6(raid5_conf_t *conf,
2521                 struct stripe_head *sh, struct stripe_head_state *s,
2522                 struct r6_state *r6s, int disks)
2523 {
2524         int rcw = 0, pd_idx = sh->pd_idx, i;
2525         int qd_idx = sh->qd_idx;
2526
2527         set_bit(STRIPE_HANDLE, &sh->state);
2528         for (i = disks; i--; ) {
2529                 struct r5dev *dev = &sh->dev[i];
2530                 /* check if we haven't enough data */
2531                 if (!test_bit(R5_OVERWRITE, &dev->flags) &&
2532                     i != pd_idx && i != qd_idx &&
2533                     !test_bit(R5_LOCKED, &dev->flags) &&
2534                     !(test_bit(R5_UPTODATE, &dev->flags) ||
2535                       test_bit(R5_Wantcompute, &dev->flags))) {
2536                         rcw++;
2537                         if (!test_bit(R5_Insync, &dev->flags))
2538                                 continue; /* it's a failed drive */
2539
2540                         if (
2541                           test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
2542                                 pr_debug("Read_old stripe %llu "
2543                                         "block %d for Reconstruct\n",
2544                                      (unsigned long long)sh->sector, i);
2545                                 set_bit(R5_LOCKED, &dev->flags);
2546                                 set_bit(R5_Wantread, &dev->flags);
2547                                 s->locked++;
2548                         } else {
2549                                 pr_debug("Request delayed stripe %llu "
2550                                         "block %d for Reconstruct\n",
2551                                      (unsigned long long)sh->sector, i);
2552                                 set_bit(STRIPE_DELAYED, &sh->state);
2553                                 set_bit(STRIPE_HANDLE, &sh->state);
2554                         }
2555                 }
2556         }
2557         /* now if nothing is locked, and if we have enough data, we can start a
2558          * write request
2559          */
2560         if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&
2561             s->locked == 0 && rcw == 0 &&
2562             !test_bit(STRIPE_BIT_DELAY, &sh->state)) {
2563                 schedule_reconstruction(sh, s, 1, 0);
2564         }
2565 }
2566
2567 static void handle_parity_checks5(raid5_conf_t *conf, struct stripe_head *sh,
2568                                 struct stripe_head_state *s, int disks)
2569 {
2570         struct r5dev *dev = NULL;
2571
2572         set_bit(STRIPE_HANDLE, &sh->state);
2573
2574         switch (sh->check_state) {
2575         case check_state_idle:
2576                 /* start a new check operation if there are no failures */
2577                 if (s->failed == 0) {
2578                         BUG_ON(s->uptodate != disks);
2579                         sh->check_state = check_state_run;
2580                         set_bit(STRIPE_OP_CHECK, &s->ops_request);
2581                         clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
2582                         s->uptodate--;
2583                         break;
2584                 }
2585                 dev = &sh->dev[s->failed_num];
2586                 /* fall through */
2587         case check_state_compute_result:
2588                 sh->check_state = check_state_idle;
2589                 if (!dev)
2590                         dev = &sh->dev[sh->pd_idx];
2591
2592                 /* check that a write has not made the stripe insync */
2593                 if (test_bit(STRIPE_INSYNC, &sh->state))
2594                         break;
2595
2596                 /* either failed parity check, or recovery is happening */
2597                 BUG_ON(!test_bit(R5_UPTODATE, &dev->flags));
2598                 BUG_ON(s->uptodate != disks);
2599
2600                 set_bit(R5_LOCKED, &dev->flags);
2601                 s->locked++;
2602                 set_bit(R5_Wantwrite, &dev->flags);
2603
2604                 clear_bit(STRIPE_DEGRADED, &sh->state);
2605                 set_bit(STRIPE_INSYNC, &sh->state);
2606                 break;
2607         case check_state_run:
2608                 break; /* we will be called again upon completion */
2609         case check_state_check_result:
2610                 sh->check_state = check_state_idle;
2611
2612                 /* if a failure occurred during the check operation, leave
2613                  * STRIPE_INSYNC not set and let the stripe be handled again
2614                  */
2615                 if (s->failed)
2616                         break;
2617
2618                 /* handle a successful check operation, if parity is correct
2619                  * we are done.  Otherwise update the mismatch count and repair
2620                  * parity if !MD_RECOVERY_CHECK
2621                  */
2622                 if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0)
2623                         /* parity is correct (on disc,
2624                          * not in buffer any more)
2625                          */
2626                         set_bit(STRIPE_INSYNC, &sh->state);
2627                 else {
2628                         conf->mddev->resync_mismatches += STRIPE_SECTORS;
2629                         if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
2630                                 /* don't try to repair!! */
2631                                 set_bit(STRIPE_INSYNC, &sh->state);
2632                         else {
2633                                 sh->check_state = check_state_compute_run;
2634                                 set_bit(STRIPE_COMPUTE_RUN, &sh->state);
2635                                 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
2636                                 set_bit(R5_Wantcompute,
2637                                         &sh->dev[sh->pd_idx].flags);
2638                                 sh->ops.target = sh->pd_idx;
2639                                 sh->ops.target2 = -1;
2640                                 s->uptodate++;
2641                         }
2642                 }
2643                 break;
2644         case check_state_compute_run:
2645                 break;
2646         default:
2647                 printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
2648                        __func__, sh->check_state,
2649                        (unsigned long long) sh->sector);
2650                 BUG();
2651         }
2652 }
2653
2654
2655 static void handle_parity_checks6(raid5_conf_t *conf, struct stripe_head *sh,
2656                                   struct stripe_head_state *s,
2657                                   struct r6_state *r6s, int disks)
2658 {
2659         int pd_idx = sh->pd_idx;
2660         int qd_idx = sh->qd_idx;
2661         struct r5dev *dev;
2662
2663         set_bit(STRIPE_HANDLE, &sh->state);
2664
2665         BUG_ON(s->failed > 2);
2666
2667         /* Want to check and possibly repair P and Q.
2668          * However there could be one 'failed' device, in which
2669          * case we can only check one of them, possibly using the
2670          * other to generate missing data
2671          */
2672
2673         switch (sh->check_state) {
2674         case check_state_idle:
2675                 /* start a new check operation if there are < 2 failures */
2676                 if (s->failed == r6s->q_failed) {
2677                         /* The only possible failed device holds Q, so it
2678                          * makes sense to check P (If anything else were failed,
2679                          * we would have used P to recreate it).
2680                          */
2681                         sh->check_state = check_state_run;
2682                 }
2683                 if (!r6s->q_failed && s->failed < 2) {
2684                         /* Q is not failed, and we didn't use it to generate
2685                          * anything, so it makes sense to check it
2686                          */
2687                         if (sh->check_state == check_state_run)
2688                                 sh->check_state = check_state_run_pq;
2689                         else
2690                                 sh->check_state = check_state_run_q;
2691                 }
2692
2693                 /* discard potentially stale zero_sum_result */
2694                 sh->ops.zero_sum_result = 0;
2695
2696                 if (sh->check_state == check_state_run) {
2697                         /* async_xor_zero_sum destroys the contents of P */
2698                         clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
2699                         s->uptodate--;
2700                 }
2701                 if (sh->check_state >= check_state_run &&
2702                     sh->check_state <= check_state_run_pq) {
2703                         /* async_syndrome_zero_sum preserves P and Q, so
2704                          * no need to mark them !uptodate here
2705                          */
2706                         set_bit(STRIPE_OP_CHECK, &s->ops_request);
2707                         break;
2708                 }
2709
2710                 /* we have 2-disk failure */
2711                 BUG_ON(s->failed != 2);
2712                 /* fall through */
2713         case check_state_compute_result:
2714                 sh->check_state = check_state_idle;
2715
2716                 /* check that a write has not made the stripe insync */
2717                 if (test_bit(STRIPE_INSYNC, &sh->state))
2718                         break;
2719
2720                 /* now write out any block on a failed drive,
2721                  * or P or Q if they were recomputed
2722                  */
2723                 BUG_ON(s->uptodate < disks - 1); /* We don't need Q to recover */
2724                 if (s->failed == 2) {
2725                         dev = &sh->dev[r6s->failed_num[1]];
2726                         s->locked++;
2727                         set_bit(R5_LOCKED, &dev->flags);
2728                         set_bit(R5_Wantwrite, &dev->flags);
2729                 }
2730                 if (s->failed >= 1) {
2731                         dev = &sh->dev[r6s->failed_num[0]];
2732                         s->locked++;
2733                         set_bit(R5_LOCKED, &dev->flags);
2734                         set_bit(R5_Wantwrite, &dev->flags);
2735                 }
2736                 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
2737                         dev = &sh->dev[pd_idx];
2738                         s->locked++;
2739                         set_bit(R5_LOCKED, &dev->flags);
2740                         set_bit(R5_Wantwrite, &dev->flags);
2741                 }
2742                 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
2743                         dev = &sh->dev[qd_idx];
2744                         s->locked++;
2745                         set_bit(R5_LOCKED, &dev->flags);
2746                         set_bit(R5_Wantwrite, &dev->flags);
2747                 }
2748                 clear_bit(STRIPE_DEGRADED, &sh->state);
2749
2750                 set_bit(STRIPE_INSYNC, &sh->state);
2751                 break;
2752         case check_state_run:
2753         case check_state_run_q:
2754         case check_state_run_pq:
2755                 break; /* we will be called again upon completion */
2756         case check_state_check_result:
2757                 sh->check_state = check_state_idle;
2758
2759                 /* handle a successful check operation, if parity is correct
2760                  * we are done.  Otherwise update the mismatch count and repair
2761                  * parity if !MD_RECOVERY_CHECK
2762                  */
2763                 if (sh->ops.zero_sum_result == 0) {
2764                         /* both parities are correct */
2765                         if (!s->failed)
2766                                 set_bit(STRIPE_INSYNC, &sh->state);
2767                         else {
2768                                 /* in contrast to the raid5 case we can validate
2769                                  * parity, but still have a failure to write
2770                                  * back
2771                                  */
2772                                 sh->check_state = check_state_compute_result;
2773                                 /* Returning at this point means that we may go
2774                                  * off and bring p and/or q uptodate again so
2775                                  * we make sure to check zero_sum_result again
2776                                  * to verify if p or q need writeback
2777                                  */
2778                         }
2779                 } else {
2780                         conf->mddev->resync_mismatches += STRIPE_SECTORS;
2781                         if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
2782                                 /* don't try to repair!! */
2783                                 set_bit(STRIPE_INSYNC, &sh->state);
2784                         else {
2785                                 int *target = &sh->ops.target;
2786
2787                                 sh->ops.target = -1;
2788                                 sh->ops.target2 = -1;
2789                                 sh->check_state = check_state_compute_run;
2790                                 set_bit(STRIPE_COMPUTE_RUN, &sh->state);
2791                                 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
2792                                 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
2793                                         set_bit(R5_Wantcompute,
2794                                                 &sh->dev[pd_idx].flags);
2795                                         *target = pd_idx;
2796                                         target = &sh->ops.target2;
2797                                         s->uptodate++;
2798                                 }
2799                                 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
2800                                         set_bit(R5_Wantcompute,
2801                                                 &sh->dev[qd_idx].flags);
2802                                         *target = qd_idx;
2803                                         s->uptodate++;
2804                                 }
2805                         }
2806                 }
2807                 break;
2808         case check_state_compute_run:
2809                 break;
2810         default:
2811                 printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
2812                        __func__, sh->check_state,
2813                        (unsigned long long) sh->sector);
2814                 BUG();
2815         }
2816 }
2817
2818 static void handle_stripe_expansion(raid5_conf_t *conf, struct stripe_head *sh,
2819                                 struct r6_state *r6s)
2820 {
2821         int i;
2822
2823         /* We have read all the blocks in this stripe and now we need to
2824          * copy some of them into a target stripe for expand.
2825          */
2826         struct dma_async_tx_descriptor *tx = NULL;
2827         clear_bit(STRIPE_EXPAND_SOURCE, &sh->state);
2828         for (i = 0; i < sh->disks; i++)
2829                 if (i != sh->pd_idx && i != sh->qd_idx) {
2830                         int dd_idx, j;
2831                         struct stripe_head *sh2;
2832                         struct async_submit_ctl submit;
2833
2834                         sector_t bn = compute_blocknr(sh, i, 1);
2835                         sector_t s = raid5_compute_sector(conf, bn, 0,
2836                                                           &dd_idx, NULL);
2837                         sh2 = get_active_stripe(conf, s, 0, 1, 1);
2838                         if (sh2 == NULL)
2839                                 /* so far only the early blocks of this stripe
2840                                  * have been requested.  When later blocks
2841                                  * get requested, we will try again
2842                                  */
2843                                 continue;
2844                         if (!test_bit(STRIPE_EXPANDING, &sh2->state) ||
2845                            test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) {
2846                                 /* must have already done this block */
2847                                 release_stripe(sh2);
2848                                 continue;
2849                         }
2850
2851                         /* place all the copies on one channel */
2852                         init_async_submit(&submit, 0, tx, NULL, NULL, NULL);
2853                         tx = async_memcpy(sh2->dev[dd_idx].page,
2854                                           sh->dev[i].page, 0, 0, STRIPE_SIZE,
2855                                           &submit);
2856
2857                         set_bit(R5_Expanded, &sh2->dev[dd_idx].flags);
2858                         set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags);
2859                         for (j = 0; j < conf->raid_disks; j++)
2860                                 if (j != sh2->pd_idx &&
2861                                     (!r6s || j != sh2->qd_idx) &&
2862                                     !test_bit(R5_Expanded, &sh2->dev[j].flags))
2863                                         break;
2864                         if (j == conf->raid_disks) {
2865                                 set_bit(STRIPE_EXPAND_READY, &sh2->state);
2866                                 set_bit(STRIPE_HANDLE, &sh2->state);
2867                         }
2868                         release_stripe(sh2);
2869
2870                 }
2871         /* done submitting copies, wait for them to complete */
2872         if (tx) {
2873                 async_tx_ack(tx);
2874                 dma_wait_for_async_tx(tx);
2875         }
2876 }
2877
2878
2879 /*
2880  * handle_stripe - do things to a stripe.
2881  *
2882  * We lock the stripe and then examine the state of various bits
2883  * to see what needs to be done.
2884  * Possible results:
2885  *    return some read request which now have data
2886  *    return some write requests which are safely on disc
2887  *    schedule a read on some buffers
2888  *    schedule a write of some buffers
2889  *    return confirmation of parity correctness
2890  *
2891  * buffers are taken off read_list or write_list, and bh_cache buffers
2892  * get BH_Lock set before the stripe lock is released.
2893  *
2894  */
2895
2896 static bool handle_stripe5(struct stripe_head *sh)
2897 {
2898         raid5_conf_t *conf = sh->raid_conf;
2899         int disks = sh->disks, i;
2900         struct bio *return_bi = NULL;
2901         struct stripe_head_state s;
2902         struct r5dev *dev;
2903         mdk_rdev_t *blocked_rdev = NULL;
2904         int prexor;
2905
2906         memset(&s, 0, sizeof(s));
2907         pr_debug("handling stripe %llu, state=%#lx cnt=%d, pd_idx=%d check:%d "
2908                  "reconstruct:%d\n", (unsigned long long)sh->sector, sh->state,
2909                  atomic_read(&sh->count), sh->pd_idx, sh->check_state,
2910                  sh->reconstruct_state);
2911
2912         spin_lock(&sh->lock);
2913         clear_bit(STRIPE_HANDLE, &sh->state);
2914         clear_bit(STRIPE_DELAYED, &sh->state);
2915
2916         s.syncing = test_bit(STRIPE_SYNCING, &sh->state);
2917         s.expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state);
2918         s.expanded = test_bit(STRIPE_EXPAND_READY, &sh->state);
2919
2920         /* Now to look around and see what can be done */
2921         rcu_read_lock();
2922         for (i=disks; i--; ) {
2923                 mdk_rdev_t *rdev;
2924                 struct r5dev *dev = &sh->dev[i];
2925                 clear_bit(R5_Insync, &dev->flags);
2926
2927                 pr_debug("check %d: state 0x%lx toread %p read %p write %p "
2928                         "written %p\n", i, dev->flags, dev->toread, dev->read,
2929                         dev->towrite, dev->written);
2930
2931                 /* maybe we can request a biofill operation
2932                  *
2933                  * new wantfill requests are only permitted while
2934                  * ops_complete_biofill is guaranteed to be inactive
2935                  */
2936                 if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread &&
2937                     !test_bit(STRIPE_BIOFILL_RUN, &sh->state))
2938                         set_bit(R5_Wantfill, &dev->flags);
2939
2940                 /* now count some things */
2941                 if (test_bit(R5_LOCKED, &dev->flags)) s.locked++;
2942                 if (test_bit(R5_UPTODATE, &dev->flags)) s.uptodate++;
2943                 if (test_bit(R5_Wantcompute, &dev->flags)) s.compute++;
2944
2945                 if (test_bit(R5_Wantfill, &dev->flags))
2946                         s.to_fill++;
2947                 else if (dev->toread)
2948                         s.to_read++;
2949                 if (dev->towrite) {
2950                         s.to_write++;
2951                         if (!test_bit(R5_OVERWRITE, &dev->flags))
2952                                 s.non_overwrite++;
2953                 }
2954                 if (dev->written)
2955                         s.written++;
2956                 rdev = rcu_dereference(conf->disks[i].rdev);
2957                 if (blocked_rdev == NULL &&
2958                     rdev && unlikely(test_bit(Blocked, &rdev->flags))) {
2959                         blocked_rdev = rdev;
2960                         atomic_inc(&rdev->nr_pending);
2961                 }
2962                 if (!rdev || !test_bit(In_sync, &rdev->flags)) {
2963                         /* The ReadError flag will just be confusing now */
2964                         clear_bit(R5_ReadError, &dev->flags);
2965                         clear_bit(R5_ReWrite, &dev->flags);
2966                 }
2967                 if (!rdev || !test_bit(In_sync, &rdev->flags)
2968                     || test_bit(R5_ReadError, &dev->flags)) {
2969                         s.failed++;
2970                         s.failed_num = i;
2971                 } else
2972                         set_bit(R5_Insync, &dev->flags);
2973         }
2974         rcu_read_unlock();
2975
2976         if (unlikely(blocked_rdev)) {
2977                 if (s.syncing || s.expanding || s.expanded ||
2978                     s.to_write || s.written) {
2979                         set_bit(STRIPE_HANDLE, &sh->state);
2980                         goto unlock;
2981                 }
2982                 /* There is nothing for the blocked_rdev to block */
2983                 rdev_dec_pending(blocked_rdev, conf->mddev);
2984                 blocked_rdev = NULL;
2985         }
2986
2987         if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) {
2988                 set_bit(STRIPE_OP_BIOFILL, &s.ops_request);
2989                 set_bit(STRIPE_BIOFILL_RUN, &sh->state);
2990         }
2991
2992         pr_debug("locked=%d uptodate=%d to_read=%d"
2993                 " to_write=%d failed=%d failed_num=%d\n",
2994                 s.locked, s.uptodate, s.to_read, s.to_write,
2995                 s.failed, s.failed_num);
2996         /* check if the array has lost two devices and, if so, some requests might
2997          * need to be failed
2998          */
2999         if (s.failed > 1 && s.to_read+s.to_write+s.written)
3000                 handle_failed_stripe(conf, sh, &s, disks, &return_bi);
3001         if (s.failed > 1 && s.syncing) {
3002                 md_done_sync(conf->mddev, STRIPE_SECTORS,0);
3003                 clear_bit(STRIPE_SYNCING, &sh->state);
3004                 s.syncing = 0;
3005         }
3006
3007         /* might be able to return some write requests if the parity block
3008          * is safe, or on a failed drive
3009          */
3010         dev = &sh->dev[sh->pd_idx];
3011         if ( s.written &&
3012              ((test_bit(R5_Insync, &dev->flags) &&
3013                !test_bit(R5_LOCKED, &dev->flags) &&
3014                test_bit(R5_UPTODATE, &dev->flags)) ||
3015                (s.failed == 1 && s.failed_num == sh->pd_idx)))
3016                 handle_stripe_clean_event(conf, sh, disks, &return_bi);
3017
3018         /* Now we might consider reading some blocks, either to check/generate
3019          * parity, or to satisfy requests
3020          * or to load a block that is being partially written.
3021          */
3022         if (s.to_read || s.non_overwrite ||
3023             (s.syncing && (s.uptodate + s.compute < disks)) || s.expanding)
3024                 handle_stripe_fill5(sh, &s, disks);
3025
3026         /* Now we check to see if any write operations have recently
3027          * completed
3028          */
3029         prexor = 0;
3030         if (sh->reconstruct_state == reconstruct_state_prexor_drain_result)
3031                 prexor = 1;
3032         if (sh->reconstruct_state == reconstruct_state_drain_result ||
3033             sh->reconstruct_state == reconstruct_state_prexor_drain_result) {
3034                 sh->reconstruct_state = reconstruct_state_idle;
3035
3036                 /* All the 'written' buffers and the parity block are ready to
3037                  * be written back to disk
3038                  */
3039                 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags));
3040                 for (i = disks; i--; ) {
3041                         dev = &sh->dev[i];
3042                         if (test_bit(R5_LOCKED, &dev->flags) &&
3043                                 (i == sh->pd_idx || dev->written)) {
3044                                 pr_debug("Writing block %d\n", i);
3045                                 set_bit(R5_Wantwrite, &dev->flags);
3046                                 if (prexor)
3047                                         continue;
3048                                 if (!test_bit(R5_Insync, &dev->flags) ||
3049                                     (i == sh->pd_idx && s.failed == 0))
3050                                         set_bit(STRIPE_INSYNC, &sh->state);
3051                         }
3052                 }
3053                 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
3054                         atomic_dec(&conf->preread_active_stripes);
3055                         if (atomic_read(&conf->preread_active_stripes) <
3056                                 IO_THRESHOLD)
3057                                 md_wakeup_thread(conf->mddev->thread);
3058                 }
3059         }
3060
3061         /* Now to consider new write requests and what else, if anything
3062          * should be read.  We do not handle new writes when:
3063          * 1/ A 'write' operation (copy+xor) is already in flight.
3064          * 2/ A 'check' operation is in flight, as it may clobber the parity
3065          *    block.
3066          */
3067         if (s.to_write && !sh->reconstruct_state && !sh->check_state)
3068                 handle_stripe_dirtying5(conf, sh, &s, disks);
3069
3070         /* maybe we need to check and possibly fix the parity for this stripe
3071          * Any reads will already have been scheduled, so we just see if enough
3072          * data is available.  The parity check is held off while parity
3073          * dependent operations are in flight.
3074          */
3075         if (sh->check_state ||
3076             (s.syncing && s.locked == 0 &&
3077              !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
3078              !test_bit(STRIPE_INSYNC, &sh->state)))
3079                 handle_parity_checks5(conf, sh, &s, disks);
3080
3081         if (s.syncing && s.locked == 0 && test_bit(STRIPE_INSYNC, &sh->state)) {
3082                 md_done_sync(conf->mddev, STRIPE_SECTORS,1);
3083                 clear_bit(STRIPE_SYNCING, &sh->state);
3084         }
3085
3086         /* If the failed drive is just a ReadError, then we might need to progress
3087          * the repair/check process
3088          */
3089         if (s.failed == 1 && !conf->mddev->ro &&
3090             test_bit(R5_ReadError, &sh->dev[s.failed_num].flags)
3091             && !test_bit(R5_LOCKED, &sh->dev[s.failed_num].flags)
3092             && test_bit(R5_UPTODATE, &sh->dev[s.failed_num].flags)
3093                 ) {
3094                 dev = &sh->dev[s.failed_num];
3095                 if (!test_bit(R5_ReWrite, &dev->flags)) {
3096                         set_bit(R5_Wantwrite, &dev->flags);
3097                         set_bit(R5_ReWrite, &dev->flags);
3098                         set_bit(R5_LOCKED, &dev->flags);
3099                         s.locked++;
3100                 } else {
3101                         /* let's read it back */
3102                         set_bit(R5_Wantread, &dev->flags);
3103                         set_bit(R5_LOCKED, &dev->flags);
3104                         s.locked++;
3105                 }
3106         }
3107
3108         /* Finish reconstruct operations initiated by the expansion process */
3109         if (sh->reconstruct_state == reconstruct_state_result) {
3110                 struct stripe_head *sh2
3111                         = get_active_stripe(conf, sh->sector, 1, 1, 1);
3112                 if (sh2 && test_bit(STRIPE_EXPAND_SOURCE, &sh2->state)) {
3113                         /* sh cannot be written until sh2 has been read.
3114                          * so arrange for sh to be delayed a little
3115                          */
3116                         set_bit(STRIPE_DELAYED, &sh->state);
3117                         set_bit(STRIPE_HANDLE, &sh->state);
3118                         if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE,
3119                                               &sh2->state))
3120                                 atomic_inc(&conf->preread_active_stripes);
3121                         release_stripe(sh2);
3122                         goto unlock;
3123                 }
3124                 if (sh2)
3125                         release_stripe(sh2);
3126
3127                 sh->reconstruct_state = reconstruct_state_idle;
3128                 clear_bit(STRIPE_EXPANDING, &sh->state);
3129                 for (i = conf->raid_disks; i--; ) {
3130                         set_bit(R5_Wantwrite, &sh->dev[i].flags);
3131                         set_bit(R5_LOCKED, &sh->dev[i].flags);
3132                         s.locked++;
3133                 }
3134         }
3135
3136         if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) &&
3137             !sh->reconstruct_state) {
3138                 /* Need to write out all blocks after computing parity */
3139                 sh->disks = conf->raid_disks;
3140                 stripe_set_idx(sh->sector, conf, 0, sh);
3141                 schedule_reconstruction(sh, &s, 1, 1);
3142         } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) {
3143                 clear_bit(STRIPE_EXPAND_READY, &sh->state);
3144                 atomic_dec(&conf->reshape_stripes);
3145                 wake_up(&conf->wait_for_overlap);
3146                 md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
3147         }
3148
3149         if (s.expanding && s.locked == 0 &&
3150             !test_bit(STRIPE_COMPUTE_RUN, &sh->state))
3151                 handle_stripe_expansion(conf, sh, NULL);
3152
3153  unlock:
3154         spin_unlock(&sh->lock);
3155
3156         /* wait for this device to become unblocked */
3157         if (unlikely(blocked_rdev))
3158                 md_wait_for_blocked_rdev(blocked_rdev, conf->mddev);
3159
3160         if (s.ops_request)
3161                 raid_run_ops(sh, s.ops_request);
3162
3163         ops_run_io(sh, &s);
3164
3165         return_io(return_bi);
3166
3167         return blocked_rdev == NULL;
3168 }
3169
3170 static bool handle_stripe6(struct stripe_head *sh)
3171 {
3172         raid5_conf_t *conf = sh->raid_conf;
3173         int disks = sh->disks;
3174         struct bio *return_bi = NULL;
3175         int i, pd_idx = sh->pd_idx, qd_idx = sh->qd_idx;
3176         struct stripe_head_state s;
3177         struct r6_state r6s;
3178         struct r5dev *dev, *pdev, *qdev;
3179         mdk_rdev_t *blocked_rdev = NULL;
3180
3181         pr_debug("handling stripe %llu, state=%#lx cnt=%d, "
3182                 "pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n",
3183                (unsigned long long)sh->sector, sh->state,
3184                atomic_read(&sh->count), pd_idx, qd_idx,
3185                sh->check_state, sh->reconstruct_state);
3186         memset(&s, 0, sizeof(s));
3187
3188         spin_lock(&sh->lock);
3189         clear_bit(STRIPE_HANDLE, &sh->state);
3190         clear_bit(STRIPE_DELAYED, &sh->state);
3191
3192         s.syncing = test_bit(STRIPE_SYNCING, &sh->state);
3193         s.expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state);
3194         s.expanded = test_bit(STRIPE_EXPAND_READY, &sh->state);
3195         /* Now to look around and see what can be done */
3196
3197         rcu_read_lock();
3198         for (i=disks; i--; ) {
3199                 mdk_rdev_t *rdev;
3200                 dev = &sh->dev[i];
3201                 clear_bit(R5_Insync, &dev->flags);
3202
3203                 pr_debug("check %d: state 0x%lx read %p write %p written %p\n",
3204                         i, dev->flags, dev->toread, dev->towrite, dev->written);
3205                 /* maybe we can reply to a read
3206                  *
3207                  * new wantfill requests are only permitted while
3208                  * ops_complete_biofill is guaranteed to be inactive
3209                  */
3210                 if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread &&
3211                     !test_bit(STRIPE_BIOFILL_RUN, &sh->state))
3212                         set_bit(R5_Wantfill, &dev->flags);
3213
3214                 /* now count some things */
3215                 if (test_bit(R5_LOCKED, &dev->flags)) s.locked++;
3216                 if (test_bit(R5_UPTODATE, &dev->flags)) s.uptodate++;
3217                 if (test_bit(R5_Wantcompute, &dev->flags))
3218                         BUG_ON(++s.compute > 2);
3219
3220                 if (test_bit(R5_Wantfill, &dev->flags)) {
3221                         s.to_fill++;
3222                 } else if (dev->toread)
3223                         s.to_read++;
3224                 if (dev->towrite) {
3225                         s.to_write++;
3226                         if (!test_bit(R5_OVERWRITE, &dev->flags))
3227                                 s.non_overwrite++;
3228                 }
3229                 if (dev->written)
3230                         s.written++;
3231                 rdev = rcu_dereference(conf->disks[i].rdev);
3232                 if (blocked_rdev == NULL &&
3233                     rdev && unlikely(test_bit(Blocked, &rdev->flags))) {
3234                         blocked_rdev = rdev;
3235                         atomic_inc(&rdev->nr_pending);
3236                 }
3237                 if (!rdev || !test_bit(In_sync, &rdev->flags)) {
3238                         /* The ReadError flag will just be confusing now */
3239                         clear_bit(R5_ReadError, &dev->flags);
3240                         clear_bit(R5_ReWrite, &dev->flags);
3241                 }
3242                 if (!rdev || !test_bit(In_sync, &rdev->flags)
3243                     || test_bit(R5_ReadError, &dev->flags)) {
3244                         if (s.failed < 2)
3245                                 r6s.failed_num[s.failed] = i;
3246                         s.failed++;
3247                 } else
3248                         set_bit(R5_Insync, &dev->flags);
3249         }
3250         rcu_read_unlock();
3251
3252         if (unlikely(blocked_rdev)) {
3253                 if (s.syncing || s.expanding || s.expanded ||
3254                     s.to_write || s.written) {
3255                         set_bit(STRIPE_HANDLE, &sh->state);
3256                         goto unlock;
3257                 }
3258                 /* There is nothing for the blocked_rdev to block */
3259                 rdev_dec_pending(blocked_rdev, conf->mddev);
3260                 blocked_rdev = NULL;
3261         }
3262
3263         if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) {
3264                 set_bit(STRIPE_OP_BIOFILL, &s.ops_request);
3265                 set_bit(STRIPE_BIOFILL_RUN, &sh->state);
3266         }
3267
3268         pr_debug("locked=%d uptodate=%d to_read=%d"
3269                " to_write=%d failed=%d failed_num=%d,%d\n",
3270                s.locked, s.uptodate, s.to_read, s.to_write, s.failed,
3271                r6s.failed_num[0], r6s.failed_num[1]);
3272         /* check if the array has lost >2 devices and, if so, some requests
3273          * might need to be failed
3274          */
3275         if (s.failed > 2 && s.to_read+s.to_write+s.written)
3276                 handle_failed_stripe(conf, sh, &s, disks, &return_bi);
3277         if (s.failed > 2 && s.syncing) {
3278                 md_done_sync(conf->mddev, STRIPE_SECTORS,0);
3279                 clear_bit(STRIPE_SYNCING, &sh->state);
3280                 s.syncing = 0;
3281         }
3282
3283         /*
3284          * might be able to return some write requests if the parity blocks
3285          * are safe, or on a failed drive
3286          */
3287         pdev = &sh->dev[pd_idx];
3288         r6s.p_failed = (s.failed >= 1 && r6s.failed_num[0] == pd_idx)
3289                 || (s.failed >= 2 && r6s.failed_num[1] == pd_idx);
3290         qdev = &sh->dev[qd_idx];
3291         r6s.q_failed = (s.failed >= 1 && r6s.failed_num[0] == qd_idx)
3292                 || (s.failed >= 2 && r6s.failed_num[1] == qd_idx);
3293
3294         if ( s.written &&
3295              ( r6s.p_failed || ((test_bit(R5_Insync, &pdev->flags)
3296                              && !test_bit(R5_LOCKED, &pdev->flags)
3297                              && test_bit(R5_UPTODATE, &pdev->flags)))) &&
3298              ( r6s.q_failed || ((test_bit(R5_Insync, &qdev->flags)
3299                              && !test_bit(R5_LOCKED, &qdev->flags)
3300                              && test_bit(R5_UPTODATE, &qdev->flags)))))
3301                 handle_stripe_clean_event(conf, sh, disks, &return_bi);
3302
3303         /* Now we might consider reading some blocks, either to check/generate
3304          * parity, or to satisfy requests
3305          * or to load a block that is being partially written.
3306          */
3307         if (s.to_read || s.non_overwrite || (s.to_write && s.failed) ||
3308             (s.syncing && (s.uptodate + s.compute < disks)) || s.expanding)
3309                 handle_stripe_fill6(sh, &s, &r6s, disks);
3310
3311         /* Now we check to see if any write operations have recently
3312          * completed
3313          */
3314         if (sh->reconstruct_state == reconstruct_state_drain_result) {
3315                 int qd_idx = sh->qd_idx;
3316
3317                 sh->reconstruct_state = reconstruct_state_idle;
3318                 /* All the 'written' buffers and the parity blocks are ready to
3319                  * be written back to disk
3320                  */
3321                 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags));
3322                 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[qd_idx].flags));
3323                 for (i = disks; i--; ) {
3324                         dev = &sh->dev[i];
3325                         if (test_bit(R5_LOCKED, &dev->flags) &&
3326                             (i == sh->pd_idx || i == qd_idx ||
3327                              dev->written)) {
3328                                 pr_debug("Writing block %d\n", i);
3329                                 BUG_ON(!test_bit(R5_UPTODATE, &dev->flags));
3330                                 set_bit(R5_Wantwrite, &dev->flags);
3331                                 if (!test_bit(R5_Insync, &dev->flags) ||
3332                                     ((i == sh->pd_idx || i == qd_idx) &&
3333                                       s.failed == 0))
3334                                         set_bit(STRIPE_INSYNC, &sh->state);
3335                         }
3336                 }
3337                 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
3338                         atomic_dec(&conf->preread_active_stripes);
3339                         if (atomic_read(&conf->preread_active_stripes) <
3340                                 IO_THRESHOLD)
3341                                 md_wakeup_thread(conf->mddev->thread);
3342                 }
3343         }
3344
3345         /* Now to consider new write requests and what else, if anything
3346          * should be read.  We do not handle new writes when:
3347          * 1/ A 'write' operation (copy+gen_syndrome) is already in flight.
3348          * 2/ A 'check' operation is in flight, as it may clobber the parity
3349          *    block.
3350          */
3351         if (s.to_write && !sh->reconstruct_state && !sh->check_state)
3352                 handle_stripe_dirtying6(conf, sh, &s, &r6s, disks);
3353
3354         /* maybe we need to check and possibly fix the parity for this stripe
3355          * Any reads will already have been scheduled, so we just see if enough
3356          * data is available.  The parity check is held off while parity
3357          * dependent operations are in flight.
3358          */
3359         if (sh->check_state ||
3360             (s.syncing && s.locked == 0 &&
3361              !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
3362              !test_bit(STRIPE_INSYNC, &sh->state)))
3363                 handle_parity_checks6(conf, sh, &s, &r6s, disks);
3364
3365         if (s.syncing && s.locked == 0 && test_bit(STRIPE_INSYNC, &sh->state)) {
3366                 md_done_sync(conf->mddev, STRIPE_SECTORS,1);
3367                 clear_bit(STRIPE_SYNCING, &sh->state);
3368         }
3369
3370         /* If the failed drives are just a ReadError, then we might need
3371          * to progress the repair/check process
3372          */
3373         if (s.failed <= 2 && !conf->mddev->ro)
3374                 for (i = 0; i < s.failed; i++) {
3375                         dev = &sh->dev[r6s.failed_num[i]];
3376                         if (test_bit(R5_ReadError, &dev->flags)
3377                             && !test_bit(R5_LOCKED, &dev->flags)
3378                             && test_bit(R5_UPTODATE, &dev->flags)
3379                                 ) {
3380                                 if (!test_bit(R5_ReWrite, &dev->flags)) {
3381                                         set_bit(R5_Wantwrite, &dev->flags);
3382                                         set_bit(R5_ReWrite, &dev->flags);
3383                                         set_bit(R5_LOCKED, &dev->flags);
3384                                         s.locked++;
3385                                 } else {
3386                                         /* let's read it back */
3387                                         set_bit(R5_Wantread, &dev->flags);
3388                                         set_bit(R5_LOCKED, &dev->flags);
3389                                         s.locked++;
3390                                 }
3391                         }
3392                 }
3393
3394         /* Finish reconstruct operations initiated by the expansion process */
3395         if (sh->reconstruct_state == reconstruct_state_result) {
3396                 sh->reconstruct_state = reconstruct_state_idle;
3397                 clear_bit(STRIPE_EXPANDING, &sh->state);
3398                 for (i = conf->raid_disks; i--; ) {
3399                         set_bit(R5_Wantwrite, &sh->dev[i].flags);
3400                         set_bit(R5_LOCKED, &sh->dev[i].flags);
3401                         s.locked++;
3402                 }
3403         }
3404
3405         if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) &&
3406             !sh->reconstruct_state) {
3407                 struct stripe_head *sh2
3408                         = get_active_stripe(conf, sh->sector, 1, 1, 1);
3409                 if (sh2 && test_bit(STRIPE_EXPAND_SOURCE, &sh2->state)) {
3410                         /* sh cannot be written until sh2 has been read.
3411                          * so arrange for sh to be delayed a little
3412                          */
3413                         set_bit(STRIPE_DELAYED, &sh->state);
3414                         set_bit(STRIPE_HANDLE, &sh->state);
3415                         if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE,
3416                                               &sh2->state))
3417                                 atomic_inc(&conf->preread_active_stripes);
3418                         release_stripe(sh2);
3419                         goto unlock;
3420                 }
3421                 if (sh2)
3422                         release_stripe(sh2);
3423
3424                 /* Need to write out all blocks after computing P&Q */
3425                 sh->disks = conf->raid_disks;
3426                 stripe_set_idx(sh->sector, conf, 0, sh);
3427                 schedule_reconstruction(sh, &s, 1, 1);
3428         } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) {
3429                 clear_bit(STRIPE_EXPAND_READY, &sh->state);
3430                 atomic_dec(&conf->reshape_stripes);
3431                 wake_up(&conf->wait_for_overlap);
3432                 md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
3433         }
3434
3435         if (s.expanding && s.locked == 0 &&
3436             !test_bit(STRIPE_COMPUTE_RUN, &sh->state))
3437                 handle_stripe_expansion(conf, sh, &r6s);
3438
3439  unlock:
3440         spin_unlock(&sh->lock);
3441
3442         /* wait for this device to become unblocked */
3443         if (unlikely(blocked_rdev))
3444                 md_wait_for_blocked_rdev(blocked_rdev, conf->mddev);
3445
3446         if (s.ops_request)
3447                 raid_run_ops(sh, s.ops_request);
3448
3449         ops_run_io(sh, &s);
3450
3451         return_io(return_bi);
3452
3453         return blocked_rdev == NULL;
3454 }
3455
3456 /* returns true if the stripe was handled */
3457 static bool handle_stripe(struct stripe_head *sh)
3458 {
3459         if (sh->raid_conf->level == 6)
3460                 return handle_stripe6(sh);
3461         else
3462                 return handle_stripe5(sh);
3463 }
3464
3465 static void raid5_activate_delayed(raid5_conf_t *conf)
3466 {
3467         if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) {
3468                 while (!list_empty(&conf->delayed_list)) {
3469                         struct list_head *l = conf->delayed_list.next;
3470                         struct stripe_head *sh;
3471                         sh = list_entry(l, struct stripe_head, lru);
3472                         list_del_init(l);
3473                         clear_bit(STRIPE_DELAYED, &sh->state);
3474                         if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
3475                                 atomic_inc(&conf->preread_active_stripes);
3476                         list_add_tail(&sh->lru, &conf->hold_list);
3477                 }
3478         } else
3479                 blk_plug_device(conf->mddev->queue);
3480 }
3481
3482 static void activate_bit_delay(raid5_conf_t *conf)
3483 {
3484         /* device_lock is held */
3485         struct list_head head;
3486         list_add(&head, &conf->bitmap_list);
3487         list_del_init(&conf->bitmap_list);
3488         while (!list_empty(&head)) {
3489                 struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru);
3490                 list_del_init(&sh->lru);
3491                 atomic_inc(&sh->count);
3492                 __release_stripe(conf, sh);
3493         }
3494 }
3495
3496 static void unplug_slaves(mddev_t *mddev)
3497 {
3498         raid5_conf_t *conf = mddev->private;
3499         int i;
3500
3501         rcu_read_lock();
3502         for (i = 0; i < conf->raid_disks; i++) {
3503                 mdk_rdev_t *rdev = rcu_dereference(conf->disks[i].rdev);
3504                 if (rdev && !test_bit(Faulty, &rdev->flags) && atomic_read(&rdev->nr_pending)) {
3505                         struct request_queue *r_queue = bdev_get_queue(rdev->bdev);
3506
3507                         atomic_inc(&rdev->nr_pending);
3508                         rcu_read_unlock();
3509
3510                         blk_unplug(r_queue);
3511
3512                         rdev_dec_pending(rdev, mddev);
3513                         rcu_read_lock();
3514                 }
3515         }
3516         rcu_read_unlock();
3517 }
3518
3519 static void raid5_unplug_device(struct request_queue *q)
3520 {
3521         mddev_t *mddev = q->queuedata;
3522         raid5_conf_t *conf = mddev->private;
3523         unsigned long flags;
3524
3525         spin_lock_irqsave(&conf->device_lock, flags);
3526
3527         if (blk_remove_plug(q)) {
3528                 conf->seq_flush++;
3529                 raid5_activate_delayed(conf);
3530         }
3531         md_wakeup_thread(mddev->thread);
3532
3533         spin_unlock_irqrestore(&conf->device_lock, flags);
3534
3535         unplug_slaves(mddev);
3536 }
3537
3538 static int raid5_congested(void *data, int bits)
3539 {
3540         mddev_t *mddev = data;
3541         raid5_conf_t *conf = mddev->private;
3542
3543         /* No difference between reads and writes.  Just check
3544          * how busy the stripe_cache is
3545          */
3546         if (conf->inactive_blocked)
3547                 return 1;
3548         if (conf->quiesce)
3549                 return 1;
3550         if (list_empty_careful(&conf->inactive_list))
3551                 return 1;
3552
3553         return 0;
3554 }
3555
3556 /* We want read requests to align with chunks where possible,
3557  * but write requests don't need to.
3558  */
3559 static int raid5_mergeable_bvec(struct request_queue *q,
3560                                 struct bvec_merge_data *bvm,
3561                                 struct bio_vec *biovec)
3562 {
3563         mddev_t *mddev = q->queuedata;
3564         sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev);
3565         int max;
3566         unsigned int chunk_sectors = mddev->chunk_sectors;
3567         unsigned int bio_sectors = bvm->bi_size >> 9;
3568
3569         if ((bvm->bi_rw & 1) == WRITE)
3570                 return biovec->bv_len; /* always allow writes to be mergeable */
3571
3572         if (mddev->new_chunk_sectors < mddev->chunk_sectors)
3573                 chunk_sectors = mddev->new_chunk_sectors;
3574         max =  (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9;
3575         if (max < 0) max = 0;
3576         if (max <= biovec->bv_len && bio_sectors == 0)
3577                 return biovec->bv_len;
3578         else
3579                 return max;
3580 }
3581
3582
3583 static int in_chunk_boundary(mddev_t *mddev, struct bio *bio)
3584 {
3585         sector_t sector = bio->bi_sector + get_start_sect(bio->bi_bdev);
3586         unsigned int chunk_sectors = mddev->chunk_sectors;
3587         unsigned int bio_sectors = bio->bi_size >> 9;
3588
3589         if (mddev->new_chunk_sectors < mddev->chunk_sectors)
3590                 chunk_sectors = mddev->new_chunk_sectors;
3591         return  chunk_sectors >=
3592                 ((sector & (chunk_sectors - 1)) + bio_sectors);
3593 }
3594
3595 /*
3596  *  add bio to the retry LIFO  ( in O(1) ... we are in interrupt )
3597  *  later sampled by raid5d.
3598  */
3599 static void add_bio_to_retry(struct bio *bi,raid5_conf_t *conf)
3600 {
3601         unsigned long flags;
3602
3603         spin_lock_irqsave(&conf->device_lock, flags);
3604
3605         bi->bi_next = conf->retry_read_aligned_list;
3606         conf->retry_read_aligned_list = bi;
3607
3608         spin_unlock_irqrestore(&conf->device_lock, flags);
3609         md_wakeup_thread(conf->mddev->thread);
3610 }
3611
3612
3613 static struct bio *remove_bio_from_retry(raid5_conf_t *conf)
3614 {
3615         struct bio *bi;
3616
3617         bi = conf->retry_read_aligned;
3618         if (bi) {
3619                 conf->retry_read_aligned = NULL;
3620                 return bi;
3621         }
3622         bi = conf->retry_read_aligned_list;
3623         if(bi) {
3624                 conf->retry_read_aligned_list = bi->bi_next;
3625                 bi->bi_next = NULL;
3626                 /*
3627                  * this sets the active strip count to 1 and the processed
3628                  * strip count to zero (upper 8 bits)
3629                  */
3630                 bi->bi_phys_segments = 1; /* biased count of active stripes */
3631         }
3632
3633         return bi;
3634 }
3635
3636
3637 /*
3638  *  The "raid5_align_endio" should check if the read succeeded and if it
3639  *  did, call bio_endio on the original bio (having bio_put the new bio
3640  *  first).
3641  *  If the read failed..
3642  */
3643 static void raid5_align_endio(struct bio *bi, int error)
3644 {
3645         struct bio* raid_bi  = bi->bi_private;
3646         mddev_t *mddev;
3647         raid5_conf_t *conf;
3648         int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
3649         mdk_rdev_t *rdev;
3650
3651         bio_put(bi);
3652
3653         mddev = raid_bi->bi_bdev->bd_disk->queue->queuedata;
3654         conf = mddev->private;
3655         rdev = (void*)raid_bi->bi_next;
3656         raid_bi->bi_next = NULL;
3657
3658         rdev_dec_pending(rdev, conf->mddev);
3659
3660         if (!error && uptodate) {
3661                 bio_endio(raid_bi, 0);
3662                 if (atomic_dec_and_test(&conf->active_aligned_reads))
3663                         wake_up(&conf->wait_for_stripe);
3664                 return;
3665         }
3666
3667
3668         pr_debug("raid5_align_endio : io error...handing IO for a retry\n");
3669
3670         add_bio_to_retry(raid_bi, conf);
3671 }
3672
3673 static int bio_fits_rdev(struct bio *bi)
3674 {
3675         struct request_queue *q = bdev_get_queue(bi->bi_bdev);
3676
3677         if ((bi->bi_size>>9) > queue_max_sectors(q))
3678                 return 0;
3679         blk_recount_segments(q, bi);
3680         if (bi->bi_phys_segments > queue_max_phys_segments(q))
3681                 return 0;
3682
3683         if (q->merge_bvec_fn)
3684                 /* it's too hard to apply the merge_bvec_fn at this stage,
3685                  * just just give up
3686                  */
3687                 return 0;
3688
3689         return 1;
3690 }
3691
3692
3693 static int chunk_aligned_read(struct request_queue *q, struct bio * raid_bio)
3694 {
3695         mddev_t *mddev = q->queuedata;
3696         raid5_conf_t *conf = mddev->private;
3697         unsigned int dd_idx;
3698         struct bio* align_bi;
3699         mdk_rdev_t *rdev;
3700
3701         if (!in_chunk_boundary(mddev, raid_bio)) {
3702                 pr_debug("chunk_aligned_read : non aligned\n");
3703                 return 0;
3704         }
3705         /*
3706          * use bio_clone to make a copy of the bio
3707          */
3708         align_bi = bio_clone(raid_bio, GFP_NOIO);
3709         if (!align_bi)
3710                 return 0;
3711         /*
3712          *   set bi_end_io to a new function, and set bi_private to the
3713          *     original bio.
3714          */
3715         align_bi->bi_end_io  = raid5_align_endio;
3716         align_bi->bi_private = raid_bio;
3717         /*
3718          *      compute position
3719          */
3720         align_bi->bi_sector =  raid5_compute_sector(conf, raid_bio->bi_sector,
3721                                                     0,
3722                                                     &dd_idx, NULL);
3723
3724         rcu_read_lock();
3725         rdev = rcu_dereference(conf->disks[dd_idx].rdev);
3726         if (rdev && test_bit(In_sync, &rdev->flags)) {
3727                 atomic_inc(&rdev->nr_pending);
3728                 rcu_read_unlock();
3729                 raid_bio->bi_next = (void*)rdev;
3730                 align_bi->bi_bdev =  rdev->bdev;
3731                 align_bi->bi_flags &= ~(1 << BIO_SEG_VALID);
3732                 align_bi->bi_sector += rdev->data_offset;
3733
3734                 if (!bio_fits_rdev(align_bi)) {
3735                         /* too big in some way */
3736                         bio_put(align_bi);
3737                         rdev_dec_pending(rdev, mddev);
3738                         return 0;
3739                 }
3740
3741                 spin_lock_irq(&conf->device_lock);
3742                 wait_event_lock_irq(conf->wait_for_stripe,
3743                                     conf->quiesce == 0,
3744                                     conf->device_lock, /* nothing */);
3745                 atomic_inc(&conf->active_aligned_reads);
3746                 spin_unlock_irq(&conf->device_lock);
3747
3748                 generic_make_request(align_bi);
3749                 return 1;
3750         } else {
3751                 rcu_read_unlock();
3752                 bio_put(align_bi);
3753                 return 0;
3754         }
3755 }
3756
3757 /* __get_priority_stripe - get the next stripe to process
3758  *
3759  * Full stripe writes are allowed to pass preread active stripes up until
3760  * the bypass_threshold is exceeded.  In general the bypass_count
3761  * increments when the handle_list is handled before the hold_list; however, it
3762  * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a
3763  * stripe with in flight i/o.  The bypass_count will be reset when the
3764  * head of the hold_list has changed, i.e. the head was promoted to the
3765  * handle_list.
3766  */
3767 static struct stripe_head *__get_priority_stripe(raid5_conf_t *conf)
3768 {
3769         struct stripe_head *sh;
3770
3771         pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n",
3772                   __func__,
3773                   list_empty(&conf->handle_list) ? "empty" : "busy",
3774                   list_empty(&conf->hold_list) ? "empty" : "busy",
3775                   atomic_read(&conf->pending_full_writes), conf->bypass_count);
3776
3777         if (!list_empty(&conf->handle_list)) {
3778                 sh = list_entry(conf->handle_list.next, typeof(*sh), lru);
3779
3780                 if (list_empty(&conf->hold_list))
3781                         conf->bypass_count = 0;
3782                 else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) {
3783                         if (conf->hold_list.next == conf->last_hold)
3784                                 conf->bypass_count++;
3785                         else {
3786                                 conf->last_hold = conf->hold_list.next;
3787                                 conf->bypass_count -= conf->bypass_threshold;
3788                                 if (conf->bypass_count < 0)
3789                                         conf->bypass_count = 0;
3790                         }
3791                 }
3792         } else if (!list_empty(&conf->hold_list) &&
3793                    ((conf->bypass_threshold &&
3794                      conf->bypass_count > conf->bypass_threshold) ||
3795                     atomic_read(&conf->pending_full_writes) == 0)) {
3796                 sh = list_entry(conf->hold_list.next,
3797                                 typeof(*sh), lru);
3798                 conf->bypass_count -= conf->bypass_threshold;
3799                 if (conf->bypass_count < 0)
3800                         conf->bypass_count = 0;
3801         } else
3802                 return NULL;
3803
3804         list_del_init(&sh->lru);
3805         atomic_inc(&sh->count);
3806         BUG_ON(atomic_read(&sh->count) != 1);
3807         return sh;
3808 }
3809
3810 static int make_request(struct request_queue *q, struct bio * bi)
3811 {
3812         mddev_t *mddev = q->queuedata;
3813         raid5_conf_t *conf = mddev->private;
3814         int dd_idx;
3815         sector_t new_sector;
3816         sector_t logical_sector, last_sector;
3817         struct stripe_head *sh;
3818         const int rw = bio_data_dir(bi);
3819         int cpu, remaining;
3820
3821         if (unlikely(bio_barrier(bi))) {
3822                 bio_endio(bi, -EOPNOTSUPP);
3823                 return 0;
3824         }
3825
3826         md_write_start(mddev, bi);
3827
3828         cpu = part_stat_lock();
3829         part_stat_inc(cpu, &mddev->gendisk->part0, ios[rw]);
3830         part_stat_add(cpu, &mddev->gendisk->part0, sectors[rw],
3831                       bio_sectors(bi));
3832         part_stat_unlock();
3833
3834         if (rw == READ &&
3835              mddev->reshape_position == MaxSector &&
3836              chunk_aligned_read(q,bi))
3837                 return 0;
3838
3839         logical_sector = bi->bi_sector & ~((sector_t)STRIPE_SECTORS-1);
3840         last_sector = bi->bi_sector + (bi->bi_size>>9);
3841         bi->bi_next = NULL;
3842         bi->bi_phys_segments = 1;       /* over-loaded to count active stripes */
3843
3844         for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) {
3845                 DEFINE_WAIT(w);
3846                 int disks, data_disks;
3847                 int previous;
3848
3849         retry:
3850                 previous = 0;
3851                 disks = conf->raid_disks;
3852                 prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE);
3853                 if (unlikely(conf->reshape_progress != MaxSector)) {
3854                         /* spinlock is needed as reshape_progress may be
3855                          * 64bit on a 32bit platform, and so it might be
3856                          * possible to see a half-updated value
3857                          * Ofcourse reshape_progress could change after
3858                          * the lock is dropped, so once we get a reference
3859                          * to the stripe that we think it is, we will have
3860                          * to check again.
3861                          */
3862                         spin_lock_irq(&conf->device_lock);
3863                         if (mddev->delta_disks < 0
3864                             ? logical_sector < conf->reshape_progress
3865                             : logical_sector >= conf->reshape_progress) {
3866                                 disks = conf->previous_raid_disks;
3867                                 previous = 1;
3868                         } else {
3869                                 if (mddev->delta_disks < 0
3870                                     ? logical_sector < conf->reshape_safe
3871                                     : logical_sector >= conf->reshape_safe) {
3872                                         spin_unlock_irq(&conf->device_lock);
3873                                         schedule();
3874                                         goto retry;
3875                                 }
3876                         }
3877                         spin_unlock_irq(&conf->device_lock);
3878                 }
3879                 data_disks = disks - conf->max_degraded;
3880
3881                 new_sector = raid5_compute_sector(conf, logical_sector,
3882                                                   previous,
3883                                                   &dd_idx, NULL);
3884                 pr_debug("raid5: make_request, sector %llu logical %llu\n",
3885                         (unsigned long long)new_sector,
3886                         (unsigned long long)logical_sector);
3887
3888                 sh = get_active_stripe(conf, new_sector, previous,
3889                                        (bi->bi_rw&RWA_MASK), 0);
3890                 if (sh) {
3891                         if (unlikely(previous)) {
3892                                 /* expansion might have moved on while waiting for a
3893                                  * stripe, so we must do the range check again.
3894                                  * Expansion could still move past after this
3895                                  * test, but as we are holding a reference to
3896                                  * 'sh', we know that if that happens,
3897                                  *  STRIPE_EXPANDING will get set and the expansion
3898                                  * won't proceed until we finish with the stripe.
3899                                  */
3900                                 int must_retry = 0;
3901                                 spin_lock_irq(&conf->device_lock);
3902                                 if (mddev->delta_disks < 0
3903                                     ? logical_sector >= conf->reshape_progress
3904                                     : logical_sector < conf->reshape_progress)
3905                                         /* mismatch, need to try again */
3906                                         must_retry = 1;
3907                                 spin_unlock_irq(&conf->device_lock);
3908                                 if (must_retry) {
3909                                         release_stripe(sh);
3910                                         schedule();
3911                                         goto retry;
3912                                 }
3913                         }
3914
3915                         if (bio_data_dir(bi) == WRITE &&
3916                             logical_sector >= mddev->suspend_lo &&
3917                             logical_sector < mddev->suspend_hi) {
3918                                 release_stripe(sh);
3919                                 /* As the suspend_* range is controlled by
3920                                  * userspace, we want an interruptible
3921                                  * wait.
3922                                  */
3923                                 flush_signals(current);
3924                                 prepare_to_wait(&conf->wait_for_overlap,
3925                                                 &w, TASK_INTERRUPTIBLE);
3926                                 if (logical_sector >= mddev->suspend_lo &&
3927                                     logical_sector < mddev->suspend_hi)
3928                                         schedule();
3929                                 goto retry;
3930                         }
3931
3932                         if (test_bit(STRIPE_EXPANDING, &sh->state) ||
3933                             !add_stripe_bio(sh, bi, dd_idx, (bi->bi_rw&RW_MASK))) {
3934                                 /* Stripe is busy expanding or
3935                                  * add failed due to overlap.  Flush everything
3936                                  * and wait a while
3937                                  */
3938                                 raid5_unplug_device(mddev->queue);
3939                                 release_stripe(sh);
3940                                 schedule();
3941                                 goto retry;
3942                         }
3943                         finish_wait(&conf->wait_for_overlap, &w);
3944                         set_bit(STRIPE_HANDLE, &sh->state);
3945                         clear_bit(STRIPE_DELAYED, &sh->state);
3946                         release_stripe(sh);
3947                 } else {
3948                         /* cannot get stripe for read-ahead, just give-up */
3949                         clear_bit(BIO_UPTODATE, &bi->bi_flags);
3950                         finish_wait(&conf->wait_for_overlap, &w);
3951                         break;
3952                 }
3953
3954         }
3955         spin_lock_irq(&conf->device_lock);
3956         remaining = raid5_dec_bi_phys_segments(bi);
3957         spin_unlock_irq(&conf->device_lock);
3958         if (remaining == 0) {
3959
3960                 if ( rw == WRITE )
3961                         md_write_end(mddev);
3962
3963                 bio_endio(bi, 0);
3964         }
3965         return 0;
3966 }
3967
3968 static sector_t raid5_size(mddev_t *mddev, sector_t sectors, int raid_disks);
3969
3970 static sector_t reshape_request(mddev_t *mddev, sector_t sector_nr, int *skipped)
3971 {
3972         /* reshaping is quite different to recovery/resync so it is
3973          * handled quite separately ... here.
3974          *
3975          * On each call to sync_request, we gather one chunk worth of
3976          * destination stripes and flag them as expanding.
3977          * Then we find all the source stripes and request reads.
3978          * As the reads complete, handle_stripe will copy the data
3979          * into the destination stripe and release that stripe.
3980          */
3981         raid5_conf_t *conf = (raid5_conf_t *) mddev->private;
3982         struct stripe_head *sh;
3983         sector_t first_sector, last_sector;
3984         int raid_disks = conf->previous_raid_disks;
3985         int data_disks = raid_disks - conf->max_degraded;
3986         int new_data_disks = conf->raid_disks - conf->max_degraded;
3987         int i;
3988         int dd_idx;
3989         sector_t writepos, readpos, safepos;
3990         sector_t stripe_addr;
3991         int reshape_sectors;
3992         struct list_head stripes;
3993
3994         if (sector_nr == 0) {
3995                 /* If restarting in the middle, skip the initial sectors */
3996                 if (mddev->delta_disks < 0 &&
3997                     conf->reshape_progress < raid5_size(mddev, 0, 0)) {
3998                         sector_nr = raid5_size(mddev, 0, 0)
3999                                 - conf->reshape_progress;
4000                 } else if (mddev->delta_disks >= 0 &&
4001                            conf->reshape_progress > 0)
4002                         sector_nr = conf->reshape_progress;
4003                 sector_div(sector_nr, new_data_disks);
4004                 if (sector_nr) {
4005                         *skipped = 1;
4006                         return sector_nr;
4007                 }
4008         }
4009
4010         /* We need to process a full chunk at a time.
4011          * If old and new chunk sizes differ, we need to process the
4012          * largest of these
4013          */
4014         if (mddev->new_chunk_sectors > mddev->chunk_sectors)
4015                 reshape_sectors = mddev->new_chunk_sectors;
4016         else
4017                 reshape_sectors = mddev->chunk_sectors;
4018
4019         /* we update the metadata when there is more than 3Meg
4020          * in the block range (that is rather arbitrary, should
4021          * probably be time based) or when the data about to be
4022          * copied would over-write the source of the data at
4023          * the front of the range.
4024          * i.e. one new_stripe along from reshape_progress new_maps
4025          * to after where reshape_safe old_maps to
4026          */
4027         writepos = conf->reshape_progress;
4028         sector_div(writepos, new_data_disks);
4029         readpos = conf->reshape_progress;
4030         sector_div(readpos, data_disks);
4031         safepos = conf->reshape_safe;
4032         sector_div(safepos, data_disks);
4033         if (mddev->delta_disks < 0) {
4034                 writepos -= min_t(sector_t, reshape_sectors, writepos);
4035                 readpos += reshape_sectors;
4036                 safepos += reshape_sectors;
4037         } else {
4038                 writepos += reshape_sectors;
4039                 readpos -= min_t(sector_t, reshape_sectors, readpos);
4040                 safepos -= min_t(sector_t, reshape_sectors, safepos);
4041         }
4042
4043         /* 'writepos' is the most advanced device address we might write.
4044          * 'readpos' is the least advanced device address we might read.
4045          * 'safepos' is the least address recorded in the metadata as having
4046          *     been reshaped.
4047          * If 'readpos' is behind 'writepos', then there is no way that we can
4048          * ensure safety in the face of a crash - that must be done by userspace
4049          * making a backup of the data.  So in that case there is no particular
4050          * rush to update metadata.
4051          * Otherwise if 'safepos' is behind 'writepos', then we really need to
4052          * update the metadata to advance 'safepos' to match 'readpos' so that
4053          * we can be safe in the event of a crash.
4054          * So we insist on updating metadata if safepos is behind writepos and
4055          * readpos is beyond writepos.
4056          * In any case, update the metadata every 10 seconds.
4057          * Maybe that number should be configurable, but I'm not sure it is
4058          * worth it.... maybe it could be a multiple of safemode_delay???
4059          */
4060         if ((mddev->delta_disks < 0
4061              ? (safepos > writepos && readpos < writepos)
4062              : (safepos < writepos && readpos > writepos)) ||
4063             time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
4064                 /* Cannot proceed until we've updated the superblock... */
4065                 wait_event(conf->wait_for_overlap,
4066                            atomic_read(&conf->reshape_stripes)==0);
4067                 mddev->reshape_position = conf->reshape_progress;
4068                 mddev->curr_resync_completed = mddev->curr_resync;
4069                 conf->reshape_checkpoint = jiffies;
4070                 set_bit(MD_CHANGE_DEVS, &mddev->flags);
4071                 md_wakeup_thread(mddev->thread);
4072                 wait_event(mddev->sb_wait, mddev->flags == 0 ||
4073                            kthread_should_stop());
4074                 spin_lock_irq(&conf->device_lock);
4075                 conf->reshape_safe = mddev->reshape_position;
4076                 spin_unlock_irq(&conf->device_lock);
4077                 wake_up(&conf->wait_for_overlap);
4078                 sysfs_notify(&mddev->kobj, NULL, "sync_completed");
4079         }
4080
4081         if (mddev->delta_disks < 0) {
4082                 BUG_ON(conf->reshape_progress == 0);
4083                 stripe_addr = writepos;
4084                 BUG_ON((mddev->dev_sectors &
4085                         ~((sector_t)reshape_sectors - 1))
4086                        - reshape_sectors - stripe_addr
4087                        != sector_nr);
4088         } else {
4089                 BUG_ON(writepos != sector_nr + reshape_sectors);
4090                 stripe_addr = sector_nr;
4091         }
4092         INIT_LIST_HEAD(&stripes);
4093         for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) {
4094                 int j;
4095                 int skipped = 0;
4096                 sh = get_active_stripe(conf, stripe_addr+i, 0, 0, 1);
4097                 set_bit(STRIPE_EXPANDING, &sh->state);
4098                 atomic_inc(&conf->reshape_stripes);
4099                 /* If any of this stripe is beyond the end of the old
4100                  * array, then we need to zero those blocks
4101                  */
4102                 for (j=sh->disks; j--;) {
4103                         sector_t s;
4104                         if (j == sh->pd_idx)
4105                                 continue;
4106                         if (conf->level == 6 &&
4107                             j == sh->qd_idx)
4108                                 continue;
4109                         s = compute_blocknr(sh, j, 0);
4110                         if (s < raid5_size(mddev, 0, 0)) {
4111                                 skipped = 1;
4112                                 continue;
4113                         }
4114                         memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE);
4115                         set_bit(R5_Expanded, &sh->dev[j].flags);
4116                         set_bit(R5_UPTODATE, &sh->dev[j].flags);
4117                 }
4118                 if (!skipped) {
4119                         set_bit(STRIPE_EXPAND_READY, &sh->state);
4120                         set_bit(STRIPE_HANDLE, &sh->state);
4121                 }
4122                 list_add(&sh->lru, &stripes);
4123         }
4124         spin_lock_irq(&conf->device_lock);
4125         if (mddev->delta_disks < 0)
4126                 conf->reshape_progress -= reshape_sectors * new_data_disks;
4127         else
4128                 conf->reshape_progress += reshape_sectors * new_data_disks;
4129         spin_unlock_irq(&conf->device_lock);
4130         /* Ok, those stripe are ready. We can start scheduling
4131          * reads on the source stripes.
4132          * The source stripes are determined by mapping the first and last
4133          * block on the destination stripes.
4134          */
4135         first_sector =
4136                 raid5_compute_sector(conf, stripe_addr*(new_data_disks),
4137                                      1, &dd_idx, NULL);
4138         last_sector =
4139                 raid5_compute_sector(conf, ((stripe_addr+reshape_sectors)
4140                                             * new_data_disks - 1),
4141                                      1, &dd_idx, NULL);
4142         if (last_sector >= mddev->dev_sectors)
4143                 last_sector = mddev->dev_sectors - 1;
4144         while (first_sector <= last_sector) {
4145                 sh = get_active_stripe(conf, first_sector, 1, 0, 1);
4146                 set_bit(STRIPE_EXPAND_SOURCE, &sh->state);
4147                 set_bit(STRIPE_HANDLE, &sh->state);
4148                 release_stripe(sh);
4149                 first_sector += STRIPE_SECTORS;
4150         }
4151         /* Now that the sources are clearly marked, we can release
4152          * the destination stripes
4153          */
4154         while (!list_empty(&stripes)) {
4155                 sh = list_entry(stripes.next, struct stripe_head, lru);
4156                 list_del_init(&sh->lru);
4157                 release_stripe(sh);
4158         }
4159         /* If this takes us to the resync_max point where we have to pause,
4160          * then we need to write out the superblock.
4161          */
4162         sector_nr += reshape_sectors;
4163         if ((sector_nr - mddev->curr_resync_completed) * 2
4164             >= mddev->resync_max - mddev->curr_resync_completed) {
4165                 /* Cannot proceed until we've updated the superblock... */
4166                 wait_event(conf->wait_for_overlap,
4167                            atomic_read(&conf->reshape_stripes) == 0);
4168                 mddev->reshape_position = conf->reshape_progress;
4169                 mddev->curr_resync_completed = mddev->curr_resync + reshape_sectors;
4170                 conf->reshape_checkpoint = jiffies;
4171                 set_bit(MD_CHANGE_DEVS, &mddev->flags);
4172                 md_wakeup_thread(mddev->thread);
4173                 wait_event(mddev->sb_wait,
4174                            !test_bit(MD_CHANGE_DEVS, &mddev->flags)
4175                            || kthread_should_stop());
4176                 spin_lock_irq(&conf->device_lock);
4177                 conf->reshape_safe = mddev->reshape_position;
4178                 spin_unlock_irq(&conf->device_lock);
4179                 wake_up(&conf->wait_for_overlap);
4180                 sysfs_notify(&mddev->kobj, NULL, "sync_completed");
4181         }
4182         return reshape_sectors;
4183 }
4184
4185 /* FIXME go_faster isn't used */
4186 static inline sector_t sync_request(mddev_t *mddev, sector_t sector_nr, int *skipped, int go_faster)
4187 {
4188         raid5_conf_t *conf = (raid5_conf_t *) mddev->private;
4189         struct stripe_head *sh;
4190         sector_t max_sector = mddev->dev_sectors;
4191         int sync_blocks;
4192         int still_degraded = 0;
4193         int i;
4194
4195         if (sector_nr >= max_sector) {
4196                 /* just being told to finish up .. nothing much to do */
4197                 unplug_slaves(mddev);
4198
4199                 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
4200                         end_reshape(conf);
4201                         return 0;
4202                 }
4203
4204                 if (mddev->curr_resync < max_sector) /* aborted */
4205                         bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
4206                                         &sync_blocks, 1);
4207                 else /* completed sync */
4208                         conf->fullsync = 0;
4209                 bitmap_close_sync(mddev->bitmap);
4210
4211                 return 0;
4212         }
4213
4214         /* Allow raid5_quiesce to complete */
4215         wait_event(conf->wait_for_overlap, conf->quiesce != 2);
4216
4217         if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
4218                 return reshape_request(mddev, sector_nr, skipped);
4219
4220         /* No need to check resync_max as we never do more than one
4221          * stripe, and as resync_max will always be on a chunk boundary,
4222          * if the check in md_do_sync didn't fire, there is no chance
4223          * of overstepping resync_max here
4224          */
4225
4226         /* if there is too many failed drives and we are trying
4227          * to resync, then assert that we are finished, because there is
4228          * nothing we can do.
4229          */
4230         if (mddev->degraded >= conf->max_degraded &&
4231             test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
4232                 sector_t rv = mddev->dev_sectors - sector_nr;
4233                 *skipped = 1;
4234                 return rv;
4235         }
4236         if (!bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) &&
4237             !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
4238             !conf->fullsync && sync_blocks >= STRIPE_SECTORS) {
4239                 /* we can skip this block, and probably more */
4240                 sync_blocks /= STRIPE_SECTORS;
4241                 *skipped = 1;
4242                 return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */
4243         }
4244
4245
4246         bitmap_cond_end_sync(mddev->bitmap, sector_nr);
4247
4248         sh = get_active_stripe(conf, sector_nr, 0, 1, 0);
4249         if (sh == NULL) {
4250                 sh = get_active_stripe(conf, sector_nr, 0, 0, 0);
4251                 /* make sure we don't swamp the stripe cache if someone else
4252                  * is trying to get access
4253                  */
4254                 schedule_timeout_uninterruptible(1);
4255         }
4256         /* Need to check if array will still be degraded after recovery/resync
4257          * We don't need to check the 'failed' flag as when that gets set,
4258          * recovery aborts.
4259          */
4260         for (i = 0; i < conf->raid_disks; i++)
4261                 if (conf->disks[i].rdev == NULL)
4262                         still_degraded = 1;
4263
4264         bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded);
4265
4266         spin_lock(&sh->lock);
4267         set_bit(STRIPE_SYNCING, &sh->state);
4268         clear_bit(STRIPE_INSYNC, &sh->state);
4269         spin_unlock(&sh->lock);
4270
4271         /* wait for any blocked device to be handled */
4272         while (unlikely(!handle_stripe(sh)))
4273                 ;
4274         release_stripe(sh);
4275
4276         return STRIPE_SECTORS;
4277 }
4278
4279 static int  retry_aligned_read(raid5_conf_t *conf, struct bio *raid_bio)
4280 {
4281         /* We may not be able to submit a whole bio at once as there
4282          * may not be enough stripe_heads available.
4283          * We cannot pre-allocate enough stripe_heads as we may need
4284          * more than exist in the cache (if we allow ever large chunks).
4285          * So we do one stripe head at a time and record in
4286          * ->bi_hw_segments how many have been done.
4287          *
4288          * We *know* that this entire raid_bio is in one chunk, so
4289          * it will be only one 'dd_idx' and only need one call to raid5_compute_sector.
4290          */
4291         struct stripe_head *sh;
4292         int dd_idx;
4293         sector_t sector, logical_sector, last_sector;
4294         int scnt = 0;
4295         int remaining;
4296         int handled = 0;
4297
4298         logical_sector = raid_bio->bi_sector & ~((sector_t)STRIPE_SECTORS-1);
4299         sector = raid5_compute_sector(conf, logical_sector,
4300                                       0, &dd_idx, NULL);
4301         last_sector = raid_bio->bi_sector + (raid_bio->bi_size>>9);
4302
4303         for (; logical_sector < last_sector;
4304              logical_sector += STRIPE_SECTORS,
4305                      sector += STRIPE_SECTORS,
4306                      scnt++) {
4307
4308                 if (scnt < raid5_bi_hw_segments(raid_bio))
4309                         /* already done this stripe */
4310                         continue;
4311
4312                 sh = get_active_stripe(conf, sector, 0, 1, 0);
4313
4314                 if (!sh) {
4315                         /* failed to get a stripe - must wait */
4316                         raid5_set_bi_hw_segments(raid_bio, scnt);
4317                         conf->retry_read_aligned = raid_bio;
4318                         return handled;
4319                 }
4320
4321                 set_bit(R5_ReadError, &sh->dev[dd_idx].flags);
4322                 if (!add_stripe_bio(sh, raid_bio, dd_idx, 0)) {
4323                         release_stripe(sh);
4324                         raid5_set_bi_hw_segments(raid_bio, scnt);
4325                         conf->retry_read_aligned = raid_bio;
4326                         return handled;
4327                 }
4328
4329                 handle_stripe(sh);
4330                 release_stripe(sh);
4331                 handled++;
4332         }
4333         spin_lock_irq(&conf->device_lock);
4334         remaining = raid5_dec_bi_phys_segments(raid_bio);
4335         spin_unlock_irq(&conf->device_lock);
4336         if (remaining == 0)
4337                 bio_endio(raid_bio, 0);
4338         if (atomic_dec_and_test(&conf->active_aligned_reads))
4339                 wake_up(&conf->wait_for_stripe);
4340         return handled;
4341 }
4342
4343 #ifdef CONFIG_MULTICORE_RAID456
4344 static void __process_stripe(void *param, async_cookie_t cookie)
4345 {
4346         struct stripe_head *sh = param;
4347
4348         handle_stripe(sh);
4349         release_stripe(sh);
4350 }
4351
4352 static void process_stripe(struct stripe_head *sh, struct list_head *domain)
4353 {
4354         async_schedule_domain(__process_stripe, sh, domain);
4355 }
4356
4357 static void synchronize_stripe_processing(struct list_head *domain)
4358 {
4359         async_synchronize_full_domain(domain);
4360 }
4361 #else
4362 static void process_stripe(struct stripe_head *sh, struct list_head *domain)
4363 {
4364         handle_stripe(sh);
4365         release_stripe(sh);
4366         cond_resched();
4367 }
4368
4369 static void synchronize_stripe_processing(struct list_head *domain)
4370 {
4371 }
4372 #endif
4373
4374
4375 /*
4376  * This is our raid5 kernel thread.
4377  *
4378  * We scan the hash table for stripes which can be handled now.
4379  * During the scan, completed stripes are saved for us by the interrupt
4380  * handler, so that they will not have to wait for our next wakeup.
4381  */
4382 static void raid5d(mddev_t *mddev)
4383 {
4384         struct stripe_head *sh;
4385         raid5_conf_t *conf = mddev->private;
4386         int handled;
4387         LIST_HEAD(raid_domain);
4388
4389         pr_debug("+++ raid5d active\n");
4390
4391         md_check_recovery(mddev);
4392
4393         handled = 0;
4394         spin_lock_irq(&conf->device_lock);
4395         while (1) {
4396                 struct bio *bio;
4397
4398                 if (conf->seq_flush != conf->seq_write) {
4399                         int seq = conf->seq_flush;
4400                         spin_unlock_irq(&conf->device_lock);
4401                         bitmap_unplug(mddev->bitmap);
4402                         spin_lock_irq(&conf->device_lock);
4403                         conf->seq_write = seq;
4404                         activate_bit_delay(conf);
4405                 }
4406
4407                 while ((bio = remove_bio_from_retry(conf))) {
4408                         int ok;
4409                         spin_unlock_irq(&conf->device_lock);
4410                         ok = retry_aligned_read(conf, bio);
4411                         spin_lock_irq(&conf->device_lock);
4412                         if (!ok)
4413                                 break;
4414                         handled++;
4415                 }
4416
4417                 sh = __get_priority_stripe(conf);
4418
4419                 if (!sh)
4420                         break;
4421                 spin_unlock_irq(&conf->device_lock);
4422
4423                 handled++;
4424                 process_stripe(sh, &raid_domain);
4425
4426                 spin_lock_irq(&conf->device_lock);
4427         }
4428         pr_debug("%d stripes handled\n", handled);
4429
4430         spin_unlock_irq(&conf->device_lock);
4431
4432         synchronize_stripe_processing(&raid_domain);
4433         async_tx_issue_pending_all();
4434         unplug_slaves(mddev);
4435
4436         pr_debug("--- raid5d inactive\n");
4437 }
4438
4439 static ssize_t
4440 raid5_show_stripe_cache_size(mddev_t *mddev, char *page)
4441 {
4442         raid5_conf_t *conf = mddev->private;
4443         if (conf)
4444                 return sprintf(page, "%d\n", conf->max_nr_stripes);
4445         else
4446                 return 0;
4447 }
4448
4449 static ssize_t
4450 raid5_store_stripe_cache_size(mddev_t *mddev, const char *page, size_t len)
4451 {
4452         raid5_conf_t *conf = mddev->private;
4453         unsigned long new;
4454         int err;
4455
4456         if (len >= PAGE_SIZE)
4457                 return -EINVAL;
4458         if (!conf)
4459                 return -ENODEV;
4460
4461         if (strict_strtoul(page, 10, &new))
4462                 return -EINVAL;
4463         if (new <= 16 || new > 32768)
4464                 return -EINVAL;
4465         while (new < conf->max_nr_stripes) {
4466                 if (drop_one_stripe(conf))
4467                         conf->max_nr_stripes--;
4468                 else
4469                         break;
4470         }
4471         err = md_allow_write(mddev);
4472         if (err)
4473                 return err;
4474         while (new > conf->max_nr_stripes) {
4475                 if (grow_one_stripe(conf))
4476                         conf->max_nr_stripes++;
4477                 else break;
4478         }
4479         return len;
4480 }
4481
4482 static struct md_sysfs_entry
4483 raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR,
4484                                 raid5_show_stripe_cache_size,
4485                                 raid5_store_stripe_cache_size);
4486
4487 static ssize_t
4488 raid5_show_preread_threshold(mddev_t *mddev, char *page)
4489 {
4490         raid5_conf_t *conf = mddev->private;
4491         if (conf)
4492                 return sprintf(page, "%d\n", conf->bypass_threshold);
4493         else
4494                 return 0;
4495 }
4496
4497 static ssize_t
4498 raid5_store_preread_threshold(mddev_t *mddev, const char *page, size_t len)
4499 {
4500         raid5_conf_t *conf = mddev->private;
4501         unsigned long new;
4502         if (len >= PAGE_SIZE)
4503                 return -EINVAL;
4504         if (!conf)
4505                 return -ENODEV;
4506
4507         if (strict_strtoul(page, 10, &new))
4508                 return -EINVAL;
4509         if (new > conf->max_nr_stripes)
4510                 return -EINVAL;
4511         conf->bypass_threshold = new;
4512         return len;
4513 }
4514
4515 static struct md_sysfs_entry
4516 raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold,
4517                                         S_IRUGO | S_IWUSR,
4518                                         raid5_show_preread_threshold,
4519                                         raid5_store_preread_threshold);
4520
4521 static ssize_t
4522 stripe_cache_active_show(mddev_t *mddev, char *page)
4523 {
4524         raid5_conf_t *conf = mddev->private;
4525         if (conf)
4526                 return sprintf(page, "%d\n", atomic_read(&conf->active_stripes));
4527         else
4528                 return 0;
4529 }
4530
4531 static struct md_sysfs_entry
4532 raid5_stripecache_active = __ATTR_RO(stripe_cache_active);
4533
4534 static struct attribute *raid5_attrs[] =  {
4535         &raid5_stripecache_size.attr,
4536         &raid5_stripecache_active.attr,
4537         &raid5_preread_bypass_threshold.attr,
4538         NULL,
4539 };
4540 static struct attribute_group raid5_attrs_group = {
4541         .name = NULL,
4542         .attrs = raid5_attrs,
4543 };
4544
4545 static sector_t
4546 raid5_size(mddev_t *mddev, sector_t sectors, int raid_disks)
4547 {
4548         raid5_conf_t *conf = mddev->private;
4549
4550         if (!sectors)
4551                 sectors = mddev->dev_sectors;
4552         if (!raid_disks) {
4553                 /* size is defined by the smallest of previous and new size */
4554                 if (conf->raid_disks < conf->previous_raid_disks)
4555                         raid_disks = conf->raid_disks;
4556                 else
4557                         raid_disks = conf->previous_raid_disks;
4558         }
4559
4560         sectors &= ~((sector_t)mddev->chunk_sectors - 1);
4561         sectors &= ~((sector_t)mddev->new_chunk_sectors - 1);
4562         return sectors * (raid_disks - conf->max_degraded);
4563 }
4564
4565 static void raid5_free_percpu(raid5_conf_t *conf)
4566 {
4567         struct raid5_percpu *percpu;
4568         unsigned long cpu;
4569
4570         if (!conf->percpu)
4571                 return;
4572
4573         get_online_cpus();
4574         for_each_possible_cpu(cpu) {
4575                 percpu = per_cpu_ptr(conf->percpu, cpu);
4576                 safe_put_page(percpu->spare_page);
4577                 kfree(percpu->scribble);
4578         }
4579 #ifdef CONFIG_HOTPLUG_CPU
4580         unregister_cpu_notifier(&conf->cpu_notify);
4581 #endif
4582         put_online_cpus();
4583
4584         free_percpu(conf->percpu);
4585 }
4586
4587 static void free_conf(raid5_conf_t *conf)
4588 {
4589         shrink_stripes(conf);
4590         raid5_free_percpu(conf);
4591         kfree(conf->disks);
4592         kfree(conf->stripe_hashtbl);
4593         kfree(conf);
4594 }
4595
4596 #ifdef CONFIG_HOTPLUG_CPU
4597 static int raid456_cpu_notify(struct notifier_block *nfb, unsigned long action,
4598                               void *hcpu)
4599 {
4600         raid5_conf_t *conf = container_of(nfb, raid5_conf_t, cpu_notify);
4601         long cpu = (long)hcpu;
4602         struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu);
4603
4604         switch (action) {
4605         case CPU_UP_PREPARE:
4606         case CPU_UP_PREPARE_FROZEN:
4607                 if (conf->level == 6 && !percpu->spare_page)
4608                         percpu->spare_page = alloc_page(GFP_KERNEL);
4609                 if (!percpu->scribble)
4610                         percpu->scribble = kmalloc(conf->scribble_len, GFP_KERNEL);
4611
4612                 if (!percpu->scribble ||
4613                     (conf->level == 6 && !percpu->spare_page)) {
4614                         safe_put_page(percpu->spare_page);
4615                         kfree(percpu->scribble);
4616                         pr_err("%s: failed memory allocation for cpu%ld\n",
4617                                __func__, cpu);
4618                         return NOTIFY_BAD;
4619                 }
4620                 break;
4621         case CPU_DEAD:
4622         case CPU_DEAD_FROZEN:
4623                 safe_put_page(percpu->spare_page);
4624                 kfree(percpu->scribble);
4625                 percpu->spare_page = NULL;
4626                 percpu->scribble = NULL;
4627                 break;
4628         default:
4629                 break;
4630         }
4631         return NOTIFY_OK;
4632 }
4633 #endif
4634
4635 static int raid5_alloc_percpu(raid5_conf_t *conf)
4636 {
4637         unsigned long cpu;
4638         struct page *spare_page;
4639         struct raid5_percpu *allcpus;
4640         void *scribble;
4641         int err;
4642
4643         allcpus = alloc_percpu(struct raid5_percpu);
4644         if (!allcpus)
4645                 return -ENOMEM;
4646         conf->percpu = allcpus;
4647
4648         get_online_cpus();
4649         err = 0;
4650         for_each_present_cpu(cpu) {
4651                 if (conf->level == 6) {
4652                         spare_page = alloc_page(GFP_KERNEL);
4653                         if (!spare_page) {
4654                                 err = -ENOMEM;
4655                                 break;
4656                         }
4657                         per_cpu_ptr(conf->percpu, cpu)->spare_page = spare_page;
4658                 }
4659                 scribble = kmalloc(scribble_len(conf->raid_disks), GFP_KERNEL);
4660                 if (!scribble) {
4661                         err = -ENOMEM;
4662                         break;
4663                 }
4664                 per_cpu_ptr(conf->percpu, cpu)->scribble = scribble;
4665         }
4666 #ifdef CONFIG_HOTPLUG_CPU
4667         conf->cpu_notify.notifier_call = raid456_cpu_notify;
4668         conf->cpu_notify.priority = 0;
4669         if (err == 0)
4670                 err = register_cpu_notifier(&conf->cpu_notify);
4671 #endif
4672         put_online_cpus();
4673
4674         return err;
4675 }
4676
4677 static raid5_conf_t *setup_conf(mddev_t *mddev)
4678 {
4679         raid5_conf_t *conf;
4680         int raid_disk, memory;
4681         mdk_rdev_t *rdev;
4682         struct disk_info *disk;
4683
4684         if (mddev->new_level != 5
4685             && mddev->new_level != 4
4686             && mddev->new_level != 6) {
4687                 printk(KERN_ERR "raid5: %s: raid level not set to 4/5/6 (%d)\n",
4688                        mdname(mddev), mddev->new_level);
4689                 return ERR_PTR(-EIO);
4690         }
4691         if ((mddev->new_level == 5
4692              && !algorithm_valid_raid5(mddev->new_layout)) ||
4693             (mddev->new_level == 6
4694              && !algorithm_valid_raid6(mddev->new_layout))) {
4695                 printk(KERN_ERR "raid5: %s: layout %d not supported\n",
4696                        mdname(mddev), mddev->new_layout);
4697                 return ERR_PTR(-EIO);
4698         }
4699         if (mddev->new_level == 6 && mddev->raid_disks < 4) {
4700                 printk(KERN_ERR "raid6: not enough configured devices for %s (%d, minimum 4)\n",
4701                        mdname(mddev), mddev->raid_disks);
4702                 return ERR_PTR(-EINVAL);
4703         }
4704
4705         if (!mddev->new_chunk_sectors ||
4706             (mddev->new_chunk_sectors << 9) % PAGE_SIZE ||
4707             !is_power_of_2(mddev->new_chunk_sectors)) {
4708                 printk(KERN_ERR "raid5: invalid chunk size %d for %s\n",
4709                        mddev->new_chunk_sectors << 9, mdname(mddev));
4710                 return ERR_PTR(-EINVAL);
4711         }
4712
4713         conf = kzalloc(sizeof(raid5_conf_t), GFP_KERNEL);
4714         if (conf == NULL)
4715                 goto abort;
4716
4717         conf->raid_disks = mddev->raid_disks;
4718         conf->scribble_len = scribble_len(conf->raid_disks);
4719         if (mddev->reshape_position == MaxSector)
4720                 conf->previous_raid_disks = mddev->raid_disks;
4721         else
4722                 conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks;
4723
4724         conf->disks = kzalloc(conf->raid_disks * sizeof(struct disk_info),
4725                               GFP_KERNEL);
4726         if (!conf->disks)
4727                 goto abort;
4728
4729         conf->mddev = mddev;
4730
4731         if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL)
4732                 goto abort;
4733
4734         conf->level = mddev->new_level;
4735         if (raid5_alloc_percpu(conf) != 0)
4736                 goto abort;
4737
4738         spin_lock_init(&conf->device_lock);
4739         init_waitqueue_head(&conf->wait_for_stripe);
4740         init_waitqueue_head(&conf->wait_for_overlap);
4741         INIT_LIST_HEAD(&conf->handle_list);
4742         INIT_LIST_HEAD(&conf->hold_list);
4743         INIT_LIST_HEAD(&conf->delayed_list);
4744         INIT_LIST_HEAD(&conf->bitmap_list);
4745         INIT_LIST_HEAD(&conf->inactive_list);
4746         atomic_set(&conf->active_stripes, 0);
4747         atomic_set(&conf->preread_active_stripes, 0);
4748         atomic_set(&conf->active_aligned_reads, 0);
4749         conf->bypass_threshold = BYPASS_THRESHOLD;
4750
4751         pr_debug("raid5: run(%s) called.\n", mdname(mddev));
4752
4753         list_for_each_entry(rdev, &mddev->disks, same_set) {
4754                 raid_disk = rdev->raid_disk;
4755                 if (raid_disk >= conf->raid_disks
4756                     || raid_disk < 0)
4757                         continue;
4758                 disk = conf->disks + raid_disk;
4759
4760                 disk->rdev = rdev;
4761
4762                 if (test_bit(In_sync, &rdev->flags)) {
4763                         char b[BDEVNAME_SIZE];
4764                         printk(KERN_INFO "raid5: device %s operational as raid"
4765                                 " disk %d\n", bdevname(rdev->bdev,b),
4766                                 raid_disk);
4767                 } else
4768                         /* Cannot rely on bitmap to complete recovery */
4769                         conf->fullsync = 1;
4770         }
4771
4772         conf->chunk_sectors = mddev->new_chunk_sectors;
4773         conf->level = mddev->new_level;
4774         if (conf->level == 6)
4775                 conf->max_degraded = 2;
4776         else
4777                 conf->max_degraded = 1;
4778         conf->algorithm = mddev->new_layout;
4779         conf->max_nr_stripes = NR_STRIPES;
4780         conf->reshape_progress = mddev->reshape_position;
4781         if (conf->reshape_progress != MaxSector) {
4782                 conf->prev_chunk_sectors = mddev->chunk_sectors;
4783                 conf->prev_algo = mddev->layout;
4784         }
4785
4786         memory = conf->max_nr_stripes * (sizeof(struct stripe_head) +
4787                  conf->raid_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024;
4788         if (grow_stripes(conf, conf->max_nr_stripes)) {
4789                 printk(KERN_ERR
4790                         "raid5: couldn't allocate %dkB for buffers\n", memory);
4791                 goto abort;
4792         } else
4793                 printk(KERN_INFO "raid5: allocated %dkB for %s\n",
4794                         memory, mdname(mddev));
4795
4796         conf->thread = md_register_thread(raid5d, mddev, "%s_raid5");
4797         if (!conf->thread) {
4798                 printk(KERN_ERR
4799                        "raid5: couldn't allocate thread for %s\n",
4800                        mdname(mddev));
4801                 goto abort;
4802         }
4803
4804         return conf;
4805
4806  abort:
4807         if (conf) {
4808                 free_conf(conf);
4809                 return ERR_PTR(-EIO);
4810         } else
4811                 return ERR_PTR(-ENOMEM);
4812 }
4813
4814 static int run(mddev_t *mddev)
4815 {
4816         raid5_conf_t *conf;
4817         int working_disks = 0, chunk_size;
4818         mdk_rdev_t *rdev;
4819
4820         if (mddev->recovery_cp != MaxSector)
4821                 printk(KERN_NOTICE "raid5: %s is not clean"
4822                        " -- starting background reconstruction\n",
4823                        mdname(mddev));
4824         if (mddev->reshape_position != MaxSector) {
4825                 /* Check that we can continue the reshape.
4826                  * Currently only disks can change, it must
4827                  * increase, and we must be past the point where
4828                  * a stripe over-writes itself
4829                  */
4830                 sector_t here_new, here_old;
4831                 int old_disks;
4832                 int max_degraded = (mddev->level == 6 ? 2 : 1);
4833
4834                 if (mddev->new_level != mddev->level) {
4835                         printk(KERN_ERR "raid5: %s: unsupported reshape "
4836                                "required - aborting.\n",
4837                                mdname(mddev));
4838                         return -EINVAL;
4839                 }
4840                 old_disks = mddev->raid_disks - mddev->delta_disks;
4841                 /* reshape_position must be on a new-stripe boundary, and one
4842                  * further up in new geometry must map after here in old
4843                  * geometry.
4844                  */
4845                 here_new = mddev->reshape_position;
4846                 if (sector_div(here_new, mddev->new_chunk_sectors *
4847                                (mddev->raid_disks - max_degraded))) {
4848                         printk(KERN_ERR "raid5: reshape_position not "
4849                                "on a stripe boundary\n");
4850                         return -EINVAL;
4851                 }
4852                 /* here_new is the stripe we will write to */
4853                 here_old = mddev->reshape_position;
4854                 sector_div(here_old, mddev->chunk_sectors *
4855                            (old_disks-max_degraded));
4856                 /* here_old is the first stripe that we might need to read
4857                  * from */
4858                 if (mddev->delta_disks == 0) {
4859                         /* We cannot be sure it is safe to start an in-place
4860                          * reshape.  It is only safe if user-space if monitoring
4861                          * and taking constant backups.
4862                          * mdadm always starts a situation like this in
4863                          * readonly mode so it can take control before
4864                          * allowing any writes.  So just check for that.
4865                          */
4866                         if ((here_new * mddev->new_chunk_sectors !=
4867                              here_old * mddev->chunk_sectors) ||
4868                             mddev->ro == 0) {
4869                                 printk(KERN_ERR "raid5: in-place reshape must be started"
4870                                        " in read-only mode - aborting\n");
4871                                 return -EINVAL;
4872                         }
4873                 } else if (mddev->delta_disks < 0
4874                     ? (here_new * mddev->new_chunk_sectors <=
4875                        here_old * mddev->chunk_sectors)
4876                     : (here_new * mddev->new_chunk_sectors >=
4877                        here_old * mddev->chunk_sectors)) {
4878                         /* Reading from the same stripe as writing to - bad */
4879                         printk(KERN_ERR "raid5: reshape_position too early for "
4880                                "auto-recovery - aborting.\n");
4881                         return -EINVAL;
4882                 }
4883                 printk(KERN_INFO "raid5: reshape will continue\n");
4884                 /* OK, we should be able to continue; */
4885         } else {
4886                 BUG_ON(mddev->level != mddev->new_level);
4887                 BUG_ON(mddev->layout != mddev->new_layout);
4888                 BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors);
4889                 BUG_ON(mddev->delta_disks != 0);
4890         }
4891
4892         if (mddev->private == NULL)
4893                 conf = setup_conf(mddev);
4894         else
4895                 conf = mddev->private;
4896
4897         if (IS_ERR(conf))
4898                 return PTR_ERR(conf);
4899
4900         mddev->thread = conf->thread;
4901         conf->thread = NULL;
4902         mddev->private = conf;
4903
4904         /*
4905          * 0 for a fully functional array, 1 or 2 for a degraded array.
4906          */
4907         list_for_each_entry(rdev, &mddev->disks, same_set)
4908                 if (rdev->raid_disk >= 0 &&
4909                     test_bit(In_sync, &rdev->flags))
4910                         working_disks++;
4911
4912         mddev->degraded = conf->raid_disks - working_disks;
4913
4914         if (mddev->degraded > conf->max_degraded) {
4915                 printk(KERN_ERR "raid5: not enough operational devices for %s"
4916                         " (%d/%d failed)\n",
4917                         mdname(mddev), mddev->degraded, conf->raid_disks);
4918                 goto abort;
4919         }
4920
4921         /* device size must be a multiple of chunk size */
4922         mddev->dev_sectors &= ~(mddev->chunk_sectors - 1);
4923         mddev->resync_max_sectors = mddev->dev_sectors;
4924
4925         if (mddev->degraded > 0 &&
4926             mddev->recovery_cp != MaxSector) {
4927                 if (mddev->ok_start_degraded)
4928                         printk(KERN_WARNING
4929                                "raid5: starting dirty degraded array: %s"
4930                                "- data corruption possible.\n",
4931                                mdname(mddev));
4932                 else {
4933                         printk(KERN_ERR
4934                                "raid5: cannot start dirty degraded array for %s\n",
4935                                mdname(mddev));
4936                         goto abort;
4937                 }
4938         }
4939
4940         if (mddev->degraded == 0)
4941                 printk("raid5: raid level %d set %s active with %d out of %d"
4942                        " devices, algorithm %d\n", conf->level, mdname(mddev),
4943                        mddev->raid_disks-mddev->degraded, mddev->raid_disks,
4944                        mddev->new_layout);
4945         else
4946                 printk(KERN_ALERT "raid5: raid level %d set %s active with %d"
4947                         " out of %d devices, algorithm %d\n", conf->level,
4948                         mdname(mddev), mddev->raid_disks - mddev->degraded,
4949                         mddev->raid_disks, mddev->new_layout);
4950
4951         print_raid5_conf(conf);
4952
4953         if (conf->reshape_progress != MaxSector) {
4954                 printk("...ok start reshape thread\n");
4955                 conf->reshape_safe = conf->reshape_progress;
4956                 atomic_set(&conf->reshape_stripes, 0);
4957                 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
4958                 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
4959                 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
4960                 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
4961                 mddev->sync_thread = md_register_thread(md_do_sync, mddev,
4962                                                         "%s_reshape");
4963         }
4964
4965         /* read-ahead size must cover two whole stripes, which is
4966          * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
4967          */
4968         {
4969                 int data_disks = conf->previous_raid_disks - conf->max_degraded;
4970                 int stripe = data_disks *
4971                         ((mddev->chunk_sectors << 9) / PAGE_SIZE);
4972                 if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
4973                         mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
4974         }
4975
4976         /* Ok, everything is just fine now */
4977         if (sysfs_create_group(&mddev->kobj, &raid5_attrs_group))
4978                 printk(KERN_WARNING
4979                        "raid5: failed to create sysfs attributes for %s\n",
4980                        mdname(mddev));
4981
4982         mddev->queue->queue_lock = &conf->device_lock;
4983
4984         mddev->queue->unplug_fn = raid5_unplug_device;
4985         mddev->queue->backing_dev_info.congested_data = mddev;
4986         mddev->queue->backing_dev_info.congested_fn = raid5_congested;
4987
4988         md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
4989
4990         blk_queue_merge_bvec(mddev->queue, raid5_mergeable_bvec);
4991         chunk_size = mddev->chunk_sectors << 9;
4992         blk_queue_io_min(mddev->queue, chunk_size);
4993         blk_queue_io_opt(mddev->queue, chunk_size *
4994                          (conf->raid_disks - conf->max_degraded));
4995
4996         list_for_each_entry(rdev, &mddev->disks, same_set)
4997                 disk_stack_limits(mddev->gendisk, rdev->bdev,
4998                                   rdev->data_offset << 9);
4999
5000         return 0;
5001 abort:
5002         md_unregister_thread(mddev->thread);
5003         mddev->thread = NULL;
5004         if (conf) {
5005                 print_raid5_conf(conf);
5006                 free_conf(conf);
5007         }
5008         mddev->private = NULL;
5009         printk(KERN_ALERT "raid5: failed to run raid set %s\n", mdname(mddev));
5010         return -EIO;
5011 }
5012
5013
5014
5015 static int stop(mddev_t *mddev)
5016 {
5017         raid5_conf_t *conf = (raid5_conf_t *) mddev->private;
5018
5019         md_unregister_thread(mddev->thread);
5020         mddev->thread = NULL;
5021         mddev->queue->backing_dev_info.congested_fn = NULL;
5022         blk_sync_queue(mddev->queue); /* the unplug fn references 'conf'*/
5023         sysfs_remove_group(&mddev->kobj, &raid5_attrs_group);
5024         free_conf(conf);
5025         mddev->private = NULL;
5026         return 0;
5027 }
5028
5029 #ifdef DEBUG
5030 static void print_sh(struct seq_file *seq, struct stripe_head *sh)
5031 {
5032         int i;
5033
5034         seq_printf(seq, "sh %llu, pd_idx %d, state %ld.\n",
5035                    (unsigned long long)sh->sector, sh->pd_idx, sh->state);
5036         seq_printf(seq, "sh %llu,  count %d.\n",
5037                    (unsigned long long)sh->sector, atomic_read(&sh->count));
5038         seq_printf(seq, "sh %llu, ", (unsigned long long)sh->sector);
5039         for (i = 0; i < sh->disks; i++) {
5040                 seq_printf(seq, "(cache%d: %p %ld) ",
5041                            i, sh->dev[i].page, sh->dev[i].flags);
5042         }
5043         seq_printf(seq, "\n");
5044 }
5045
5046 static void printall(struct seq_file *seq, raid5_conf_t *conf)
5047 {
5048         struct stripe_head *sh;
5049         struct hlist_node *hn;
5050         int i;
5051
5052         spin_lock_irq(&conf->device_lock);
5053         for (i = 0; i < NR_HASH; i++) {
5054                 hlist_for_each_entry(sh, hn, &conf->stripe_hashtbl[i], hash) {
5055                         if (sh->raid_conf != conf)
5056                                 continue;
5057                         print_sh(seq, sh);
5058                 }
5059         }
5060         spin_unlock_irq(&conf->device_lock);
5061 }
5062 #endif
5063
5064 static void status(struct seq_file *seq, mddev_t *mddev)
5065 {
5066         raid5_conf_t *conf = (raid5_conf_t *) mddev->private;
5067         int i;
5068
5069         seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level,
5070                 mddev->chunk_sectors / 2, mddev->layout);
5071         seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded);
5072         for (i = 0; i < conf->raid_disks; i++)
5073                 seq_printf (seq, "%s",
5074                                conf->disks[i].rdev &&
5075                                test_bit(In_sync, &conf->disks[i].rdev->flags) ? "U" : "_");
5076         seq_printf (seq, "]");
5077 #ifdef DEBUG
5078         seq_printf (seq, "\n");
5079         printall(seq, conf);
5080 #endif
5081 }
5082
5083 static void print_raid5_conf (raid5_conf_t *conf)
5084 {
5085         int i;
5086         struct disk_info *tmp;
5087
5088         printk("RAID5 conf printout:\n");
5089         if (!conf) {
5090                 printk("(conf==NULL)\n");
5091                 return;
5092         }
5093         printk(" --- rd:%d wd:%d\n", conf->raid_disks,
5094                  conf->raid_disks - conf->mddev->degraded);
5095
5096         for (i = 0; i < conf->raid_disks; i++) {
5097                 char b[BDEVNAME_SIZE];
5098                 tmp = conf->disks + i;
5099                 if (tmp->rdev)
5100                 printk(" disk %d, o:%d, dev:%s\n",
5101                         i, !test_bit(Faulty, &tmp->rdev->flags),
5102                         bdevname(tmp->rdev->bdev,b));
5103         }
5104 }
5105
5106 static int raid5_spare_active(mddev_t *mddev)
5107 {
5108         int i;
5109         raid5_conf_t *conf = mddev->private;
5110         struct disk_info *tmp;
5111
5112         for (i = 0; i < conf->raid_disks; i++) {
5113                 tmp = conf->disks + i;
5114                 if (tmp->rdev
5115                     && !test_bit(Faulty, &tmp->rdev->flags)
5116                     && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {
5117                         unsigned long flags;
5118                         spin_lock_irqsave(&conf->device_lock, flags);
5119                         mddev->degraded--;
5120                         spin_unlock_irqrestore(&conf->device_lock, flags);
5121                 }
5122         }
5123         print_raid5_conf(conf);
5124         return 0;
5125 }
5126
5127 static int raid5_remove_disk(mddev_t *mddev, int number)
5128 {
5129         raid5_conf_t *conf = mddev->private;
5130         int err = 0;
5131         mdk_rdev_t *rdev;
5132         struct disk_info *p = conf->disks + number;
5133
5134         print_raid5_conf(conf);
5135         rdev = p->rdev;
5136         if (rdev) {
5137                 if (number >= conf->raid_disks &&
5138                     conf->reshape_progress == MaxSector)
5139                         clear_bit(In_sync, &rdev->flags);
5140
5141                 if (test_bit(In_sync, &rdev->flags) ||
5142                     atomic_read(&rdev->nr_pending)) {
5143                         err = -EBUSY;
5144                         goto abort;
5145                 }
5146                 /* Only remove non-faulty devices if recovery
5147                  * isn't possible.
5148                  */
5149                 if (!test_bit(Faulty, &rdev->flags) &&
5150                     mddev->degraded <= conf->max_degraded &&
5151                     number < conf->raid_disks) {
5152                         err = -EBUSY;
5153                         goto abort;
5154                 }
5155                 p->rdev = NULL;
5156                 synchronize_rcu();
5157                 if (atomic_read(&rdev->nr_pending)) {
5158                         /* lost the race, try later */
5159                         err = -EBUSY;
5160                         p->rdev = rdev;
5161                 }
5162         }
5163 abort:
5164
5165         print_raid5_conf(conf);
5166         return err;
5167 }
5168
5169 static int raid5_add_disk(mddev_t *mddev, mdk_rdev_t *rdev)
5170 {
5171         raid5_conf_t *conf = mddev->private;
5172         int err = -EEXIST;
5173         int disk;
5174         struct disk_info *p;
5175         int first = 0;
5176         int last = conf->raid_disks - 1;
5177
5178         if (mddev->degraded > conf->max_degraded)
5179                 /* no point adding a device */
5180                 return -EINVAL;
5181
5182         if (rdev->raid_disk >= 0)
5183                 first = last = rdev->raid_disk;
5184
5185         /*
5186          * find the disk ... but prefer rdev->saved_raid_disk
5187          * if possible.
5188          */
5189         if (rdev->saved_raid_disk >= 0 &&
5190             rdev->saved_raid_disk >= first &&
5191             conf->disks[rdev->saved_raid_disk].rdev == NULL)
5192                 disk = rdev->saved_raid_disk;
5193         else
5194                 disk = first;
5195         for ( ; disk <= last ; disk++)
5196                 if ((p=conf->disks + disk)->rdev == NULL) {
5197                         clear_bit(In_sync, &rdev->flags);
5198                         rdev->raid_disk = disk;
5199                         err = 0;
5200                         if (rdev->saved_raid_disk != disk)
5201                                 conf->fullsync = 1;
5202                         rcu_assign_pointer(p->rdev, rdev);
5203                         break;
5204                 }
5205         print_raid5_conf(conf);
5206         return err;
5207 }
5208
5209 static int raid5_resize(mddev_t *mddev, sector_t sectors)
5210 {
5211         /* no resync is happening, and there is enough space
5212          * on all devices, so we can resize.
5213          * We need to make sure resync covers any new space.
5214          * If the array is shrinking we should possibly wait until
5215          * any io in the removed space completes, but it hardly seems
5216          * worth it.
5217          */
5218         sectors &= ~((sector_t)mddev->chunk_sectors - 1);
5219         md_set_array_sectors(mddev, raid5_size(mddev, sectors,
5220                                                mddev->raid_disks));
5221         if (mddev->array_sectors >
5222             raid5_size(mddev, sectors, mddev->raid_disks))
5223                 return -EINVAL;
5224         set_capacity(mddev->gendisk, mddev->array_sectors);
5225         mddev->changed = 1;
5226         revalidate_disk(mddev->gendisk);
5227         if (sectors > mddev->dev_sectors && mddev->recovery_cp == MaxSector) {
5228                 mddev->recovery_cp = mddev->dev_sectors;
5229                 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
5230         }
5231         mddev->dev_sectors = sectors;
5232         mddev->resync_max_sectors = sectors;
5233         return 0;
5234 }
5235
5236 static int check_stripe_cache(mddev_t *mddev)
5237 {
5238         /* Can only proceed if there are plenty of stripe_heads.
5239          * We need a minimum of one full stripe,, and for sensible progress
5240          * it is best to have about 4 times that.
5241          * If we require 4 times, then the default 256 4K stripe_heads will
5242          * allow for chunk sizes up to 256K, which is probably OK.
5243          * If the chunk size is greater, user-space should request more
5244          * stripe_heads first.
5245          */
5246         raid5_conf_t *conf = mddev->private;
5247         if (((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4
5248             > conf->max_nr_stripes ||
5249             ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4
5250             > conf->max_nr_stripes) {
5251                 printk(KERN_WARNING "raid5: reshape: not enough stripes.  Needed %lu\n",
5252                        ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9)
5253                         / STRIPE_SIZE)*4);
5254                 return 0;
5255         }
5256         return 1;
5257 }
5258
5259 static int check_reshape(mddev_t *mddev)
5260 {
5261         raid5_conf_t *conf = mddev->private;
5262
5263         if (mddev->delta_disks == 0 &&
5264             mddev->new_layout == mddev->layout &&
5265             mddev->new_chunk_sectors == mddev->chunk_sectors)
5266                 return 0; /* nothing to do */
5267         if (mddev->bitmap)
5268                 /* Cannot grow a bitmap yet */
5269                 return -EBUSY;
5270         if (mddev->degraded > conf->max_degraded)
5271                 return -EINVAL;
5272         if (mddev->delta_disks < 0) {
5273                 /* We might be able to shrink, but the devices must
5274                  * be made bigger first.
5275                  * For raid6, 4 is the minimum size.
5276                  * Otherwise 2 is the minimum
5277                  */
5278                 int min = 2;
5279                 if (mddev->level == 6)
5280                         min = 4;
5281                 if (mddev->raid_disks + mddev->delta_disks < min)
5282                         return -EINVAL;
5283         }
5284
5285         if (!check_stripe_cache(mddev))
5286                 return -ENOSPC;
5287
5288         return resize_stripes(conf, conf->raid_disks + mddev->delta_disks);
5289 }
5290
5291 static int raid5_start_reshape(mddev_t *mddev)
5292 {
5293         raid5_conf_t *conf = mddev->private;
5294         mdk_rdev_t *rdev;
5295         int spares = 0;
5296         int added_devices = 0;
5297         unsigned long flags;
5298
5299         if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
5300                 return -EBUSY;
5301
5302         if (!check_stripe_cache(mddev))
5303                 return -ENOSPC;
5304
5305         list_for_each_entry(rdev, &mddev->disks, same_set)
5306                 if (rdev->raid_disk < 0 &&
5307                     !test_bit(Faulty, &rdev->flags))
5308                         spares++;
5309
5310         if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded)
5311                 /* Not enough devices even to make a degraded array
5312                  * of that size
5313                  */
5314                 return -EINVAL;
5315
5316         /* Refuse to reduce size of the array.  Any reductions in
5317          * array size must be through explicit setting of array_size
5318          * attribute.
5319          */
5320         if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks)
5321             < mddev->array_sectors) {
5322                 printk(KERN_ERR "md: %s: array size must be reduced "
5323                        "before number of disks\n", mdname(mddev));
5324                 return -EINVAL;
5325         }
5326
5327         atomic_set(&conf->reshape_stripes, 0);
5328         spin_lock_irq(&conf->device_lock);
5329         conf->previous_raid_disks = conf->raid_disks;
5330         conf->raid_disks += mddev->delta_disks;
5331         conf->prev_chunk_sectors = conf->chunk_sectors;
5332         conf->chunk_sectors = mddev->new_chunk_sectors;
5333         conf->prev_algo = conf->algorithm;
5334         conf->algorithm = mddev->new_layout;
5335         if (mddev->delta_disks < 0)
5336                 conf->reshape_progress = raid5_size(mddev, 0, 0);
5337         else
5338                 conf->reshape_progress = 0;
5339         conf->reshape_safe = conf->reshape_progress;
5340         conf->generation++;
5341         spin_unlock_irq(&conf->device_lock);
5342
5343         /* Add some new drives, as many as will fit.
5344          * We know there are enough to make the newly sized array work.
5345          */
5346         list_for_each_entry(rdev, &mddev->disks, same_set)
5347                 if (rdev->raid_disk < 0 &&
5348                     !test_bit(Faulty, &rdev->flags)) {
5349                         if (raid5_add_disk(mddev, rdev) == 0) {
5350                                 char nm[20];
5351                                 set_bit(In_sync, &rdev->flags);
5352                                 added_devices++;
5353                                 rdev->recovery_offset = 0;
5354                                 sprintf(nm, "rd%d", rdev->raid_disk);
5355                                 if (sysfs_create_link(&mddev->kobj,
5356                                                       &rdev->kobj, nm))
5357                                         printk(KERN_WARNING
5358                                                "raid5: failed to create "
5359                                                " link %s for %s\n",
5360                                                nm, mdname(mddev));
5361                         } else
5362                                 break;
5363                 }
5364
5365         if (mddev->delta_disks > 0) {
5366                 spin_lock_irqsave(&conf->device_lock, flags);
5367                 mddev->degraded = (conf->raid_disks - conf->previous_raid_disks)
5368                         - added_devices;
5369                 spin_unlock_irqrestore(&conf->device_lock, flags);
5370         }
5371         mddev->raid_disks = conf->raid_disks;
5372         mddev->reshape_position = conf->reshape_progress;
5373         set_bit(MD_CHANGE_DEVS, &mddev->flags);
5374
5375         clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
5376         clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
5377         set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
5378         set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
5379         mddev->sync_thread = md_register_thread(md_do_sync, mddev,
5380                                                 "%s_reshape");
5381         if (!mddev->sync_thread) {
5382                 mddev->recovery = 0;
5383                 spin_lock_irq(&conf->device_lock);
5384                 mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks;
5385                 conf->reshape_progress = MaxSector;
5386                 spin_unlock_irq(&conf->device_lock);
5387                 return -EAGAIN;
5388         }
5389         conf->reshape_checkpoint = jiffies;
5390         md_wakeup_thread(mddev->sync_thread);
5391         md_new_event(mddev);
5392         return 0;
5393 }
5394
5395 /* This is called from the reshape thread and should make any
5396  * changes needed in 'conf'
5397  */
5398 static void end_reshape(raid5_conf_t *conf)
5399 {
5400
5401         if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) {
5402
5403                 spin_lock_irq(&conf->device_lock);
5404                 conf->previous_raid_disks = conf->raid_disks;
5405                 conf->reshape_progress = MaxSector;
5406                 spin_unlock_irq(&conf->device_lock);
5407                 wake_up(&conf->wait_for_overlap);
5408
5409                 /* read-ahead size must cover two whole stripes, which is
5410                  * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
5411                  */
5412                 {
5413                         int data_disks = conf->raid_disks - conf->max_degraded;
5414                         int stripe = data_disks * ((conf->chunk_sectors << 9)
5415                                                    / PAGE_SIZE);
5416                         if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
5417                                 conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
5418                 }
5419         }
5420 }
5421
5422 /* This is called from the raid5d thread with mddev_lock held.
5423  * It makes config changes to the device.
5424  */
5425 static void raid5_finish_reshape(mddev_t *mddev)
5426 {
5427         raid5_conf_t *conf = mddev->private;
5428
5429         if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
5430
5431                 if (mddev->delta_disks > 0) {
5432                         md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
5433                         set_capacity(mddev->gendisk, mddev->array_sectors);
5434                         mddev->changed = 1;
5435                         revalidate_disk(mddev->gendisk);
5436                 } else {
5437                         int d;
5438                         mddev->degraded = conf->raid_disks;
5439                         for (d = 0; d < conf->raid_disks ; d++)
5440                                 if (conf->disks[d].rdev &&
5441                                     test_bit(In_sync,
5442                                              &conf->disks[d].rdev->flags))
5443                                         mddev->degraded--;
5444                         for (d = conf->raid_disks ;
5445                              d < conf->raid_disks - mddev->delta_disks;
5446                              d++) {
5447                                 mdk_rdev_t *rdev = conf->disks[d].rdev;
5448                                 if (rdev && raid5_remove_disk(mddev, d) == 0) {
5449                                         char nm[20];
5450                                         sprintf(nm, "rd%d", rdev->raid_disk);
5451                                         sysfs_remove_link(&mddev->kobj, nm);
5452                                         rdev->raid_disk = -1;
5453                                 }
5454                         }
5455                 }
5456                 mddev->layout = conf->algorithm;
5457                 mddev->chunk_sectors = conf->chunk_sectors;
5458                 mddev->reshape_position = MaxSector;
5459                 mddev->delta_disks = 0;
5460         }
5461 }
5462
5463 static void raid5_quiesce(mddev_t *mddev, int state)
5464 {
5465         raid5_conf_t *conf = mddev->private;
5466
5467         switch(state) {
5468         case 2: /* resume for a suspend */
5469                 wake_up(&conf->wait_for_overlap);
5470                 break;
5471
5472         case 1: /* stop all writes */
5473                 spin_lock_irq(&conf->device_lock);
5474                 /* '2' tells resync/reshape to pause so that all
5475                  * active stripes can drain
5476                  */
5477                 conf->quiesce = 2;
5478                 wait_event_lock_irq(conf->wait_for_stripe,
5479                                     atomic_read(&conf->active_stripes) == 0 &&
5480                                     atomic_read(&conf->active_aligned_reads) == 0,
5481                                     conf->device_lock, /* nothing */);
5482                 conf->quiesce = 1;
5483                 spin_unlock_irq(&conf->device_lock);
5484                 /* allow reshape to continue */
5485                 wake_up(&conf->wait_for_overlap);
5486                 break;
5487
5488         case 0: /* re-enable writes */
5489                 spin_lock_irq(&conf->device_lock);
5490                 conf->quiesce = 0;
5491                 wake_up(&conf->wait_for_stripe);
5492                 wake_up(&conf->wait_for_overlap);
5493                 spin_unlock_irq(&conf->device_lock);
5494                 break;
5495         }
5496 }
5497
5498
5499 static void *raid5_takeover_raid1(mddev_t *mddev)
5500 {
5501         int chunksect;
5502
5503         if (mddev->raid_disks != 2 ||
5504             mddev->degraded > 1)
5505                 return ERR_PTR(-EINVAL);
5506
5507         /* Should check if there are write-behind devices? */
5508
5509         chunksect = 64*2; /* 64K by default */
5510
5511         /* The array must be an exact multiple of chunksize */
5512         while (chunksect && (mddev->array_sectors & (chunksect-1)))
5513                 chunksect >>= 1;
5514
5515         if ((chunksect<<9) < STRIPE_SIZE)
5516                 /* array size does not allow a suitable chunk size */
5517                 return ERR_PTR(-EINVAL);
5518
5519         mddev->new_level = 5;
5520         mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC;
5521         mddev->new_chunk_sectors = chunksect;
5522
5523         return setup_conf(mddev);
5524 }
5525
5526 static void *raid5_takeover_raid6(mddev_t *mddev)
5527 {
5528         int new_layout;
5529
5530         switch (mddev->layout) {
5531         case ALGORITHM_LEFT_ASYMMETRIC_6:
5532                 new_layout = ALGORITHM_LEFT_ASYMMETRIC;
5533                 break;
5534         case ALGORITHM_RIGHT_ASYMMETRIC_6:
5535                 new_layout = ALGORITHM_RIGHT_ASYMMETRIC;
5536                 break;
5537         case ALGORITHM_LEFT_SYMMETRIC_6:
5538                 new_layout = ALGORITHM_LEFT_SYMMETRIC;
5539                 break;
5540         case ALGORITHM_RIGHT_SYMMETRIC_6:
5541                 new_layout = ALGORITHM_RIGHT_SYMMETRIC;
5542                 break;
5543         case ALGORITHM_PARITY_0_6:
5544                 new_layout = ALGORITHM_PARITY_0;
5545                 break;
5546         case ALGORITHM_PARITY_N:
5547                 new_layout = ALGORITHM_PARITY_N;
5548                 break;
5549         default:
5550                 return ERR_PTR(-EINVAL);
5551         }
5552         mddev->new_level = 5;
5553         mddev->new_layout = new_layout;
5554         mddev->delta_disks = -1;
5555         mddev->raid_disks -= 1;
5556         return setup_conf(mddev);
5557 }
5558
5559
5560 static int raid5_check_reshape(mddev_t *mddev)
5561 {
5562         /* For a 2-drive array, the layout and chunk size can be changed
5563          * immediately as not restriping is needed.
5564          * For larger arrays we record the new value - after validation
5565          * to be used by a reshape pass.
5566          */
5567         raid5_conf_t *conf = mddev->private;
5568         int new_chunk = mddev->new_chunk_sectors;
5569
5570         if (mddev->new_layout >= 0 && !algorithm_valid_raid5(mddev->new_layout))
5571                 return -EINVAL;
5572         if (new_chunk > 0) {
5573                 if (!is_power_of_2(new_chunk))
5574                         return -EINVAL;
5575                 if (new_chunk < (PAGE_SIZE>>9))
5576                         return -EINVAL;
5577                 if (mddev->array_sectors & (new_chunk-1))
5578                         /* not factor of array size */
5579                         return -EINVAL;
5580         }
5581
5582         /* They look valid */
5583
5584         if (mddev->raid_disks == 2) {
5585                 /* can make the change immediately */
5586                 if (mddev->new_layout >= 0) {
5587                         conf->algorithm = mddev->new_layout;
5588                         mddev->layout = mddev->new_layout;
5589                 }
5590                 if (new_chunk > 0) {
5591                         conf->chunk_sectors = new_chunk ;
5592                         mddev->chunk_sectors = new_chunk;
5593                 }
5594                 set_bit(MD_CHANGE_DEVS, &mddev->flags);
5595                 md_wakeup_thread(mddev->thread);
5596         }
5597         return check_reshape(mddev);
5598 }
5599
5600 static int raid6_check_reshape(mddev_t *mddev)
5601 {
5602         int new_chunk = mddev->new_chunk_sectors;
5603
5604         if (mddev->new_layout >= 0 && !algorithm_valid_raid6(mddev->new_layout))
5605                 return -EINVAL;
5606         if (new_chunk > 0) {
5607                 if (!is_power_of_2(new_chunk))
5608                         return -EINVAL;
5609                 if (new_chunk < (PAGE_SIZE >> 9))
5610                         return -EINVAL;
5611                 if (mddev->array_sectors & (new_chunk-1))
5612                         /* not factor of array size */
5613                         return -EINVAL;
5614         }
5615
5616         /* They look valid */
5617         return check_reshape(mddev);
5618 }
5619
5620 static void *raid5_takeover(mddev_t *mddev)
5621 {
5622         /* raid5 can take over:
5623          *  raid0 - if all devices are the same - make it a raid4 layout
5624          *  raid1 - if there are two drives.  We need to know the chunk size
5625          *  raid4 - trivial - just use a raid4 layout.
5626          *  raid6 - Providing it is a *_6 layout
5627          */
5628
5629         if (mddev->level == 1)
5630                 return raid5_takeover_raid1(mddev);
5631         if (mddev->level == 4) {
5632                 mddev->new_layout = ALGORITHM_PARITY_N;
5633                 mddev->new_level = 5;
5634                 return setup_conf(mddev);
5635         }
5636         if (mddev->level == 6)
5637                 return raid5_takeover_raid6(mddev);
5638
5639         return ERR_PTR(-EINVAL);
5640 }
5641
5642
5643 static struct mdk_personality raid5_personality;
5644
5645 static void *raid6_takeover(mddev_t *mddev)
5646 {
5647         /* Currently can only take over a raid5.  We map the
5648          * personality to an equivalent raid6 personality
5649          * with the Q block at the end.
5650          */
5651         int new_layout;
5652
5653         if (mddev->pers != &raid5_personality)
5654                 return ERR_PTR(-EINVAL);
5655         if (mddev->degraded > 1)
5656                 return ERR_PTR(-EINVAL);
5657         if (mddev->raid_disks > 253)
5658                 return ERR_PTR(-EINVAL);
5659         if (mddev->raid_disks < 3)
5660                 return ERR_PTR(-EINVAL);
5661
5662         switch (mddev->layout) {
5663         case ALGORITHM_LEFT_ASYMMETRIC:
5664                 new_layout = ALGORITHM_LEFT_ASYMMETRIC_6;
5665                 break;
5666         case ALGORITHM_RIGHT_ASYMMETRIC:
5667                 new_layout = ALGORITHM_RIGHT_ASYMMETRIC_6;
5668                 break;
5669         case ALGORITHM_LEFT_SYMMETRIC:
5670                 new_layout = ALGORITHM_LEFT_SYMMETRIC_6;
5671                 break;
5672         case ALGORITHM_RIGHT_SYMMETRIC:
5673                 new_layout = ALGORITHM_RIGHT_SYMMETRIC_6;
5674                 break;
5675         case ALGORITHM_PARITY_0:
5676                 new_layout = ALGORITHM_PARITY_0_6;
5677                 break;
5678         case ALGORITHM_PARITY_N:
5679                 new_layout = ALGORITHM_PARITY_N;
5680                 break;
5681         default:
5682                 return ERR_PTR(-EINVAL);
5683         }
5684         mddev->new_level = 6;
5685         mddev->new_layout = new_layout;
5686         mddev->delta_disks = 1;
5687         mddev->raid_disks += 1;
5688         return setup_conf(mddev);
5689 }
5690
5691
5692 static struct mdk_personality raid6_personality =
5693 {
5694         .name           = "raid6",
5695         .level          = 6,
5696         .owner          = THIS_MODULE,
5697         .make_request   = make_request,
5698         .run            = run,
5699         .stop           = stop,
5700         .status         = status,
5701         .error_handler  = error,
5702         .hot_add_disk   = raid5_add_disk,
5703         .hot_remove_disk= raid5_remove_disk,
5704         .spare_active   = raid5_spare_active,
5705         .sync_request   = sync_request,
5706         .resize         = raid5_resize,
5707         .size           = raid5_size,
5708         .check_reshape  = raid6_check_reshape,
5709         .start_reshape  = raid5_start_reshape,
5710         .finish_reshape = raid5_finish_reshape,
5711         .quiesce        = raid5_quiesce,
5712         .takeover       = raid6_takeover,
5713 };
5714 static struct mdk_personality raid5_personality =
5715 {
5716         .name           = "raid5",
5717         .level          = 5,
5718         .owner          = THIS_MODULE,
5719         .make_request   = make_request,
5720         .run            = run,
5721         .stop           = stop,
5722         .status         = status,
5723         .error_handler  = error,
5724         .hot_add_disk   = raid5_add_disk,
5725         .hot_remove_disk= raid5_remove_disk,
5726         .spare_active   = raid5_spare_active,
5727         .sync_request   = sync_request,
5728         .resize         = raid5_resize,
5729         .size           = raid5_size,
5730         .check_reshape  = raid5_check_reshape,
5731         .start_reshape  = raid5_start_reshape,
5732         .finish_reshape = raid5_finish_reshape,
5733         .quiesce        = raid5_quiesce,
5734         .takeover       = raid5_takeover,
5735 };
5736
5737 static struct mdk_personality raid4_personality =
5738 {
5739         .name           = "raid4",
5740         .level          = 4,
5741         .owner          = THIS_MODULE,
5742         .make_request   = make_request,
5743         .run            = run,
5744         .stop           = stop,
5745         .status         = status,
5746         .error_handler  = error,
5747         .hot_add_disk   = raid5_add_disk,
5748         .hot_remove_disk= raid5_remove_disk,
5749         .spare_active   = raid5_spare_active,
5750         .sync_request   = sync_request,
5751         .resize         = raid5_resize,
5752         .size           = raid5_size,
5753         .check_reshape  = raid5_check_reshape,
5754         .start_reshape  = raid5_start_reshape,
5755         .finish_reshape = raid5_finish_reshape,
5756         .quiesce        = raid5_quiesce,
5757 };
5758
5759 static int __init raid5_init(void)
5760 {
5761         register_md_personality(&raid6_personality);
5762         register_md_personality(&raid5_personality);
5763         register_md_personality(&raid4_personality);
5764         return 0;
5765 }
5766
5767 static void raid5_exit(void)
5768 {
5769         unregister_md_personality(&raid6_personality);
5770         unregister_md_personality(&raid5_personality);
5771         unregister_md_personality(&raid4_personality);
5772 }
5773
5774 module_init(raid5_init);
5775 module_exit(raid5_exit);
5776 MODULE_LICENSE("GPL");
5777 MODULE_ALIAS("md-personality-4"); /* RAID5 */
5778 MODULE_ALIAS("md-raid5");
5779 MODULE_ALIAS("md-raid4");
5780 MODULE_ALIAS("md-level-5");
5781 MODULE_ALIAS("md-level-4");
5782 MODULE_ALIAS("md-personality-8"); /* RAID6 */
5783 MODULE_ALIAS("md-raid6");
5784 MODULE_ALIAS("md-level-6");
5785
5786 /* This used to be two separate modules, they were: */
5787 MODULE_ALIAS("raid5");
5788 MODULE_ALIAS("raid6");