drivers/mmc/host/atmel-mci.c

   1 /*
   2  * Atmel MultiMedia Card Interface driver
   3  *
   4  * Copyright (C) 2004-2008 Atmel Corporation
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  */
  10 #include <linux/blkdev.h>
  11 #include <linux/clk.h>
  12 #include <linux/debugfs.h>
  13 #include <linux/device.h>
  14 #include <linux/dmaengine.h>
  15 #include <linux/dma-mapping.h>
  16 #include <linux/err.h>
  17 #include <linux/gpio.h>
  18 #include <linux/init.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/ioport.h>
  21 #include <linux/module.h>
  22 #include <linux/platform_device.h>
  23 #include <linux/scatterlist.h>
  24 #include <linux/seq_file.h>
  25 #include <linux/stat.h>
  26
  27 #include <linux/mmc/host.h>
  28
  29 #include <mach/atmel-mci.h>
  30 #include <linux/atmel-mci.h>
  31
  32 #include <asm/io.h>
  33 #include <asm/unaligned.h>
  34
  35 #include <mach/cpu.h>
  36 #include <mach/board.h>
  37
  38 #include "atmel-mci-regs.h"
  39
  40 #define ATMCI_DATA_ERROR_FLAGS  (MCI_DCRCE | MCI_DTOE | MCI_OVRE | MCI_UNRE)
  41 #define ATMCI_DMA_THRESHOLD     16
  42
  43 enum {
  44         EVENT_CMD_COMPLETE = 0,
  45         EVENT_XFER_COMPLETE,
  46         EVENT_DATA_COMPLETE,
  47         EVENT_DATA_ERROR,
  48 };
  49
  50 enum atmel_mci_state {
  51         STATE_IDLE = 0,
  52         STATE_SENDING_CMD,
  53         STATE_SENDING_DATA,
  54         STATE_DATA_BUSY,
  55         STATE_SENDING_STOP,
  56         STATE_DATA_ERROR,
  57 };
  58
  59 struct atmel_mci_dma {
  60 #ifdef CONFIG_MMC_ATMELMCI_DMA
  61         struct dma_chan                 *chan;
  62         struct dma_async_tx_descriptor  *data_desc;
  63 #endif
  64 };
  65
  66 /**
  67  * struct atmel_mci - MMC controller state shared between all slots
  68  * @lock: Spinlock protecting the queue and associated data.
  69  * @regs: Pointer to MMIO registers.
  70  * @sg: Scatterlist entry currently being processed by PIO code, if any.
  71  * @pio_offset: Offset into the current scatterlist entry.
  72  * @cur_slot: The slot which is currently using the controller.
  73  * @mrq: The request currently being processed on @cur_slot,
  74  *      or NULL if the controller is idle.
  75  * @cmd: The command currently being sent to the card, or NULL.
  76  * @data: The data currently being transferred, or NULL if no data
  77  *      transfer is in progress.
  78  * @dma: DMA client state.
  79  * @data_chan: DMA channel being used for the current data transfer.
  80  * @cmd_status: Snapshot of SR taken upon completion of the current
  81  *      command. Only valid when EVENT_CMD_COMPLETE is pending.
  82  * @data_status: Snapshot of SR taken upon completion of the current
  83  *      data transfer. Only valid when EVENT_DATA_COMPLETE or
  84  *      EVENT_DATA_ERROR is pending.
  85  * @stop_cmdr: Value to be loaded into CMDR when the stop command is
  86  *      to be sent.
  87  * @tasklet: Tasklet running the request state machine.
  88  * @pending_events: Bitmask of events flagged by the interrupt handler
  89  *      to be processed by the tasklet.
  90  * @completed_events: Bitmask of events which the state machine has
  91  *      processed.
  92  * @state: Tasklet state.
  93  * @queue: List of slots waiting for access to the controller.
  94  * @need_clock_update: Update the clock rate before the next request.
  95  * @need_reset: Reset controller before next request.
  96  * @mode_reg: Value of the MR register.
  97  * @bus_hz: The rate of @mck in Hz. This forms the basis for MMC bus
  98  *      rate and timeout calculations.
  99  * @mapbase: Physical address of the MMIO registers.
 100  * @mck: The peripheral bus clock hooked up to the MMC controller.
 101  * @pdev: Platform device associated with the MMC controller.
 102  * @slot: Slots sharing this MMC controller.
 103  *
 104  * Locking
 105  * =======
 106  *
 107  * @lock is a softirq-safe spinlock protecting @queue as well as
 108  * @cur_slot, @mrq and @state. These must always be updated
 109  * at the same time while holding @lock.
 110  *
 111  * @lock also protects mode_reg and need_clock_update since these are
 112  * used to synchronize mode register updates with the queue
 113  * processing.
 114  *
 115  * The @mrq field of struct atmel_mci_slot is also protected by @lock,
 116  * and must always be written at the same time as the slot is added to
 117  * @queue.
 118  *
 119  * @pending_events and @completed_events are accessed using atomic bit
 120  * operations, so they don't need any locking.
 121  *
 122  * None of the fields touched by the interrupt handler need any
 123  * locking. However, ordering is important: Before EVENT_DATA_ERROR or
 124  * EVENT_DATA_COMPLETE is set in @pending_events, all data-related
 125  * interrupts must be disabled and @data_status updated with a
 126  * snapshot of SR. Similarly, before EVENT_CMD_COMPLETE is set, the
 127  * CMDRDY interupt must be disabled and @cmd_status updated with a
 128  * snapshot of SR, and before EVENT_XFER_COMPLETE can be set, the
 129  * bytes_xfered field of @data must be written. This is ensured by
 130  * using barriers.
 131  */
 132 struct atmel_mci {
 133         spinlock_t              lock;
 134         void __iomem            *regs;
 135
 136         struct scatterlist      *sg;
 137         unsigned int            pio_offset;
 138
 139         struct atmel_mci_slot   *cur_slot;
 140         struct mmc_request      *mrq;
 141         struct mmc_command      *cmd;
 142         struct mmc_data         *data;
 143
 144         struct atmel_mci_dma    dma;
 145         struct dma_chan         *data_chan;
 146
 147         u32                     cmd_status;
 148         u32                     data_status;
 149         u32                     stop_cmdr;
 150
 151         struct tasklet_struct   tasklet;
 152         unsigned long           pending_events;
 153         unsigned long           completed_events;
 154         enum atmel_mci_state    state;
 155         struct list_head        queue;
 156
 157         bool                    need_clock_update;
 158         bool                    need_reset;
 159         u32                     mode_reg;
 160         unsigned long           bus_hz;
 161         unsigned long           mapbase;
 162         struct clk              *mck;
 163         struct platform_device  *pdev;
 164
 165         struct atmel_mci_slot   *slot[ATMEL_MCI_MAX_NR_SLOTS];
 166 };
 167
 168 /**
 169  * struct atmel_mci_slot - MMC slot state
 170  * @mmc: The mmc_host representing this slot.
 171  * @host: The MMC controller this slot is using.
 172  * @sdc_reg: Value of SDCR to be written before using this slot.
 173  * @mrq: mmc_request currently being processed or waiting to be
 174  *      processed, or NULL when the slot is idle.
 175  * @queue_node: List node for placing this node in the @queue list of
 176  *      &struct atmel_mci.
 177  * @clock: Clock rate configured by set_ios(). Protected by host->lock.
 178  * @flags: Random state bits associated with the slot.
 179  * @detect_pin: GPIO pin used for card detection, or negative if not
 180  *      available.
 181  * @wp_pin: GPIO pin used for card write protect sending, or negative
 182  *      if not available.
 183  * @detect_is_active_high: The state of the detect pin when it is active.
 184  * @detect_timer: Timer used for debouncing @detect_pin interrupts.
 185  */
 186 struct atmel_mci_slot {
 187         struct mmc_host         *mmc;
 188         struct atmel_mci        *host;
 189
 190         u32                     sdc_reg;
 191
 192         struct mmc_request      *mrq;
 193         struct list_head        queue_node;
 194
 195         unsigned int            clock;
 196         unsigned long           flags;
 197 #define ATMCI_CARD_PRESENT      0
 198 #define ATMCI_CARD_NEED_INIT    1
 199 #define ATMCI_SHUTDOWN          2
 200
 201         int                     detect_pin;
 202         int                     wp_pin;
 203         bool                    detect_is_active_high;
 204
 205         struct timer_list       detect_timer;
 206 };
 207
 208 #define atmci_test_and_clear_pending(host, event)               \
 209         test_and_clear_bit(event, &host->pending_events)
 210 #define atmci_set_completed(host, event)                        \
 211         set_bit(event, &host->completed_events)
 212 #define atmci_set_pending(host, event)                          \
 213         set_bit(event, &host->pending_events)
 214
 215 /*
 216  * Enable or disable features/registers based on
 217  * whether the processor supports them
 218  */
 219 static bool mci_has_rwproof(void)
 220 {
 221         if (cpu_is_at91sam9261() || cpu_is_at91rm9200())
 222                 return false;
 223         else
 224                 return true;
 225 }
 226
 227 /*
 228  * The debugfs stuff below is mostly optimized away when
 229  * CONFIG_DEBUG_FS is not set.
 230  */
 231 static int atmci_req_show(struct seq_file *s, void *v)
 232 {
 233         struct atmel_mci_slot   *slot = s->private;
 234         struct mmc_request      *mrq;
 235         struct mmc_command      *cmd;
 236         struct mmc_command      *stop;
 237         struct mmc_data         *data;
 238
 239         /* Make sure we get a consistent snapshot */
 240         spin_lock_bh(&slot->host->lock);
 241         mrq = slot->mrq;
 242
 243         if (mrq) {
 244                 cmd = mrq->cmd;
 245                 data = mrq->data;
 246                 stop = mrq->stop;
 247
 248                 if (cmd)
 249                         seq_printf(s,
 250                                 "CMD%u(0x%x) flg %x rsp %x %x %x %x err %d\n",
 251                                 cmd->opcode, cmd->arg, cmd->flags,
 252                                 cmd->resp[0], cmd->resp[1], cmd->resp[2],
 253                                 cmd->resp[2], cmd->error);
 254                 if (data)
 255                         seq_printf(s, "DATA %u / %u * %u flg %x err %d\n",
 256                                 data->bytes_xfered, data->blocks,
 257                                 data->blksz, data->flags, data->error);
 258                 if (stop)
 259                         seq_printf(s,
 260                                 "CMD%u(0x%x) flg %x rsp %x %x %x %x err %d\n",
 261                                 stop->opcode, stop->arg, stop->flags,
 262                                 stop->resp[0], stop->resp[1], stop->resp[2],
 263                                 stop->resp[2], stop->error);
 264         }
 265
 266         spin_unlock_bh(&slot->host->lock);
 267
 268         return 0;
 269 }
 270
 271 static int atmci_req_open(struct inode *inode, struct file *file)
 272 {
 273         return single_open(file, atmci_req_show, inode->i_private);
 274 }
 275
 276 static const struct file_operations atmci_req_fops = {
 277         .owner          = THIS_MODULE,
 278         .open           = atmci_req_open,
 279         .read           = seq_read,
 280         .llseek         = seq_lseek,
 281         .release        = single_release,
 282 };
 283
 284 static void atmci_show_status_reg(struct seq_file *s,
 285                 const char *regname, u32 value)
 286 {
 287         static const char       *sr_bit[] = {
 288                 [0]     = "CMDRDY",
 289                 [1]     = "RXRDY",
 290                 [2]     = "TXRDY",
 291                 [3]     = "BLKE",
 292                 [4]     = "DTIP",
 293                 [5]     = "NOTBUSY",
 294                 [6]     = "ENDRX",
 295                 [7]     = "ENDTX",
 296                 [8]     = "SDIOIRQA",
 297                 [9]     = "SDIOIRQB",
 298                 [12]    = "SDIOWAIT",
 299                 [14]    = "RXBUFF",
 300                 [15]    = "TXBUFE",
 301                 [16]    = "RINDE",
 302                 [17]    = "RDIRE",
 303                 [18]    = "RCRCE",
 304                 [19]    = "RENDE",
 305                 [20]    = "RTOE",
 306                 [21]    = "DCRCE",
 307                 [22]    = "DTOE",
 308                 [23]    = "CSTOE",
 309                 [24]    = "BLKOVRE",
 310                 [25]    = "DMADONE",
 311                 [26]    = "FIFOEMPTY",
 312                 [27]    = "XFRDONE",
 313                 [30]    = "OVRE",
 314                 [31]    = "UNRE",
 315         };
 316         unsigned int            i;
 317
 318         seq_printf(s, "%s:\t0x%08x", regname, value);
 319         for (i = 0; i < ARRAY_SIZE(sr_bit); i++) {
 320                 if (value & (1 << i)) {
 321                         if (sr_bit[i])
 322                                 seq_printf(s, " %s", sr_bit[i]);
 323                         else
 324                                 seq_puts(s, " UNKNOWN");
 325                 }
 326         }
 327         seq_putc(s, '\n');
 328 }
 329
 330 static int atmci_regs_show(struct seq_file *s, void *v)
 331 {
 332         struct atmel_mci        *host = s->private;
 333         u32                     *buf;
 334
 335         buf = kmalloc(MCI_REGS_SIZE, GFP_KERNEL);
 336         if (!buf)
 337                 return -ENOMEM;
 338
 339         /*
 340          * Grab a more or less consistent snapshot. Note that we're
 341          * not disabling interrupts, so IMR and SR may not be
 342          * consistent.
 343          */
 344         spin_lock_bh(&host->lock);
 345         clk_enable(host->mck);
 346         memcpy_fromio(buf, host->regs, MCI_REGS_SIZE);
 347         clk_disable(host->mck);
 348         spin_unlock_bh(&host->lock);
 349
 350         seq_printf(s, "MR:\t0x%08x%s%s CLKDIV=%u\n",
 351                         buf[MCI_MR / 4],
 352                         buf[MCI_MR / 4] & MCI_MR_RDPROOF ? " RDPROOF" : "",
 353                         buf[MCI_MR / 4] & MCI_MR_WRPROOF ? " WRPROOF" : "",
 354                         buf[MCI_MR / 4] & 0xff);
 355         seq_printf(s, "DTOR:\t0x%08x\n", buf[MCI_DTOR / 4]);
 356         seq_printf(s, "SDCR:\t0x%08x\n", buf[MCI_SDCR / 4]);
 357         seq_printf(s, "ARGR:\t0x%08x\n", buf[MCI_ARGR / 4]);
 358         seq_printf(s, "BLKR:\t0x%08x BCNT=%u BLKLEN=%u\n",
 359                         buf[MCI_BLKR / 4],
 360                         buf[MCI_BLKR / 4] & 0xffff,
 361                         (buf[MCI_BLKR / 4] >> 16) & 0xffff);
 362
 363         /* Don't read RSPR and RDR; it will consume the data there */
 364
 365         atmci_show_status_reg(s, "SR", buf[MCI_SR / 4]);
 366         atmci_show_status_reg(s, "IMR", buf[MCI_IMR / 4]);
 367
 368         kfree(buf);
 369
 370         return 0;
 371 }
 372
 373 static int atmci_regs_open(struct inode *inode, struct file *file)
 374 {
 375         return single_open(file, atmci_regs_show, inode->i_private);
 376 }
 377
 378 static const struct file_operations atmci_regs_fops = {
 379         .owner          = THIS_MODULE,
 380         .open           = atmci_regs_open,
 381         .read           = seq_read,
 382         .llseek         = seq_lseek,
 383         .release        = single_release,
 384 };
 385
 386 static void atmci_init_debugfs(struct atmel_mci_slot *slot)
 387 {
 388         struct mmc_host         *mmc = slot->mmc;
 389         struct atmel_mci        *host = slot->host;
 390         struct dentry           *root;
 391         struct dentry           *node;
 392
 393         root = mmc->debugfs_root;
 394         if (!root)
 395                 return;
 396
 397         node = debugfs_create_file("regs", S_IRUSR, root, host,
 398                         &atmci_regs_fops);
 399         if (IS_ERR(node))
 400                 return;
 401         if (!node)
 402                 goto err;
 403
 404         node = debugfs_create_file("req", S_IRUSR, root, slot, &atmci_req_fops);
 405         if (!node)
 406                 goto err;
 407
 408         node = debugfs_create_u32("state", S_IRUSR, root, (u32 *)&host->state);
 409         if (!node)
 410                 goto err;
 411
 412         node = debugfs_create_x32("pending_events", S_IRUSR, root,
 413                                      (u32 *)&host->pending_events);
 414         if (!node)
 415                 goto err;
 416
 417         node = debugfs_create_x32("completed_events", S_IRUSR, root,
 418                                      (u32 *)&host->completed_events);
 419         if (!node)
 420                 goto err;
 421
 422         return;
 423
 424 err:
 425         dev_err(&mmc->class_dev, "failed to initialize debugfs for slot\n");
 426 }
 427
 428 static inline unsigned int ns_to_clocks(struct atmel_mci *host,
 429                                         unsigned int ns)
 430 {
 431         return (ns * (host->bus_hz / 1000000) + 999) / 1000;
 432 }
 433
 434 static void atmci_set_timeout(struct atmel_mci *host,
 435                 struct atmel_mci_slot *slot, struct mmc_data *data)
 436 {
 437         static unsigned dtomul_to_shift[] = {
 438                 0, 4, 7, 8, 10, 12, 16, 20
 439         };
 440         unsigned        timeout;
 441         unsigned        dtocyc;
 442         unsigned        dtomul;
 443
 444         timeout = ns_to_clocks(host, data->timeout_ns) + data->timeout_clks;
 445
 446         for (dtomul = 0; dtomul < 8; dtomul++) {
 447                 unsigned shift = dtomul_to_shift[dtomul];
 448                 dtocyc = (timeout + (1 << shift) - 1) >> shift;
 449                 if (dtocyc < 15)
 450                         break;
 451         }
 452
 453         if (dtomul >= 8) {
 454                 dtomul = 7;
 455                 dtocyc = 15;
 456         }
 457
 458         dev_vdbg(&slot->mmc->class_dev, "setting timeout to %u cycles\n",
 459                         dtocyc << dtomul_to_shift[dtomul]);
 460         mci_writel(host, DTOR, (MCI_DTOMUL(dtomul) | MCI_DTOCYC(dtocyc)));
 461 }
 462
 463 /*
 464  * Return mask with command flags to be enabled for this command.
 465  */
 466 static u32 atmci_prepare_command(struct mmc_host *mmc,
 467                                  struct mmc_command *cmd)
 468 {
 469         struct mmc_data *data;
 470         u32             cmdr;
 471
 472         cmd->error = -EINPROGRESS;
 473
 474         cmdr = MCI_CMDR_CMDNB(cmd->opcode);
 475
 476         if (cmd->flags & MMC_RSP_PRESENT) {
 477                 if (cmd->flags & MMC_RSP_136)
 478                         cmdr |= MCI_CMDR_RSPTYP_136BIT;
 479                 else
 480                         cmdr |= MCI_CMDR_RSPTYP_48BIT;
 481         }
 482
 483         /*
 484          * This should really be MAXLAT_5 for CMD2 and ACMD41, but
 485          * it's too difficult to determine whether this is an ACMD or
 486          * not. Better make it 64.
 487          */
 488         cmdr |= MCI_CMDR_MAXLAT_64CYC;
 489
 490         if (mmc->ios.bus_mode == MMC_BUSMODE_OPENDRAIN)
 491                 cmdr |= MCI_CMDR_OPDCMD;
 492
 493         data = cmd->data;
 494         if (data) {
 495                 cmdr |= MCI_CMDR_START_XFER;
 496                 if (data->flags & MMC_DATA_STREAM)
 497                         cmdr |= MCI_CMDR_STREAM;
 498                 else if (data->blocks > 1)
 499                         cmdr |= MCI_CMDR_MULTI_BLOCK;
 500                 else
 501                         cmdr |= MCI_CMDR_BLOCK;
 502
 503                 if (data->flags & MMC_DATA_READ)
 504                         cmdr |= MCI_CMDR_TRDIR_READ;
 505         }
 506
 507         return cmdr;
 508 }
 509
 510 static void atmci_start_command(struct atmel_mci *host,
 511                 struct mmc_command *cmd, u32 cmd_flags)
 512 {
 513         WARN_ON(host->cmd);
 514         host->cmd = cmd;
 515
 516         dev_vdbg(&host->pdev->dev,
 517                         "start command: ARGR=0x%08x CMDR=0x%08x\n",
 518                         cmd->arg, cmd_flags);
 519
 520         mci_writel(host, ARGR, cmd->arg);
 521         mci_writel(host, CMDR, cmd_flags);
 522 }
 523
 524 static void send_stop_cmd(struct atmel_mci *host, struct mmc_data *data)
 525 {
 526         atmci_start_command(host, data->stop, host->stop_cmdr);
 527         mci_writel(host, IER, MCI_CMDRDY);
 528 }
 529
 530 #ifdef CONFIG_MMC_ATMELMCI_DMA
 531 static void atmci_dma_cleanup(struct atmel_mci *host)
 532 {
 533         struct mmc_data                 *data = host->data;
 534
 535         dma_unmap_sg(&host->pdev->dev, data->sg, data->sg_len,
 536                      ((data->flags & MMC_DATA_WRITE)
 537                       ? DMA_TO_DEVICE : DMA_FROM_DEVICE));
 538 }
 539
 540 static void atmci_stop_dma(struct atmel_mci *host)
 541 {
 542         struct dma_chan *chan = host->data_chan;
 543
 544         if (chan) {
 545                 chan->device->device_terminate_all(chan);
 546                 atmci_dma_cleanup(host);
 547         } else {
 548                 /* Data transfer was stopped by the interrupt handler */
 549                 atmci_set_pending(host, EVENT_XFER_COMPLETE);
 550                 mci_writel(host, IER, MCI_NOTBUSY);
 551         }
 552 }
 553
 554 /* This function is called by the DMA driver from tasklet context. */
 555 static void atmci_dma_complete(void *arg)
 556 {
 557         struct atmel_mci        *host = arg;
 558         struct mmc_data         *data = host->data;
 559
 560         dev_vdbg(&host->pdev->dev, "DMA complete\n");
 561
 562         atmci_dma_cleanup(host);
 563
 564         /*
 565          * If the card was removed, data will be NULL. No point trying
 566          * to send the stop command or waiting for NBUSY in this case.
 567          */
 568         if (data) {
 569                 atmci_set_pending(host, EVENT_XFER_COMPLETE);
 570                 tasklet_schedule(&host->tasklet);
 571
 572                 /*
 573                  * Regardless of what the documentation says, we have
 574                  * to wait for NOTBUSY even after block read
 575                  * operations.
 576                  *
 577                  * When the DMA transfer is complete, the controller
 578                  * may still be reading the CRC from the card, i.e.
 579                  * the data transfer is still in progress and we
 580                  * haven't seen all the potential error bits yet.
 581                  *
 582                  * The interrupt handler will schedule a different
 583                  * tasklet to finish things up when the data transfer
 584                  * is completely done.
 585                  *
 586                  * We may not complete the mmc request here anyway
 587                  * because the mmc layer may call back and cause us to
 588                  * violate the "don't submit new operations from the
 589                  * completion callback" rule of the dma engine
 590                  * framework.
 591                  */
 592                 mci_writel(host, IER, MCI_NOTBUSY);
 593         }
 594 }
 595
 596 static int
 597 atmci_submit_data_dma(struct atmel_mci *host, struct mmc_data *data)
 598 {
 599         struct dma_chan                 *chan;
 600         struct dma_async_tx_descriptor  *desc;
 601         struct scatterlist              *sg;
 602         unsigned int                    i;
 603         enum dma_data_direction         direction;
 604         unsigned int                    sglen;
 605
 606         /*
 607          * We don't do DMA on "complex" transfers, i.e. with
 608          * non-word-aligned buffers or lengths. Also, we don't bother
 609          * with all the DMA setup overhead for short transfers.
 610          */
 611         if (data->blocks * data->blksz < ATMCI_DMA_THRESHOLD)
 612                 return -EINVAL;
 613         if (data->blksz & 3)
 614                 return -EINVAL;
 615
 616         for_each_sg(data->sg, sg, data->sg_len, i) {
 617                 if (sg->offset & 3 || sg->length & 3)
 618                         return -EINVAL;
 619         }
 620
 621         /* If we don't have a channel, we can't do DMA */
 622         chan = host->dma.chan;
 623         if (chan)
 624                 host->data_chan = chan;
 625
 626         if (!chan)
 627                 return -ENODEV;
 628
 629         if (data->flags & MMC_DATA_READ)
 630                 direction = DMA_FROM_DEVICE;
 631         else
 632                 direction = DMA_TO_DEVICE;
 633
 634         sglen = dma_map_sg(&host->pdev->dev, data->sg, data->sg_len, direction);
 635         if (sglen != data->sg_len)
 636                 goto unmap_exit;
 637         desc = chan->device->device_prep_slave_sg(chan,
 638                         data->sg, data->sg_len, direction,
 639                         DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 640         if (!desc)
 641                 goto unmap_exit;
 642
 643         host->dma.data_desc = desc;
 644         desc->callback = atmci_dma_complete;
 645         desc->callback_param = host;
 646         desc->tx_submit(desc);
 647
 648         /* Go! */
 649         chan->device->device_issue_pending(chan);
 650
 651         return 0;
 652 unmap_exit:
 653         dma_unmap_sg(&host->pdev->dev, data->sg, sglen, direction);
 654         return -ENOMEM;
 655 }
 656
 657 #else /* CONFIG_MMC_ATMELMCI_DMA */
 658
 659 static int atmci_submit_data_dma(struct atmel_mci *host, struct mmc_data *data)
 660 {
 661         return -ENOSYS;
 662 }
 663
 664 static void atmci_stop_dma(struct atmel_mci *host)
 665 {
 666         /* Data transfer was stopped by the interrupt handler */
 667         atmci_set_pending(host, EVENT_XFER_COMPLETE);
 668         mci_writel(host, IER, MCI_NOTBUSY);
 669 }
 670
 671 #endif /* CONFIG_MMC_ATMELMCI_DMA */
 672
 673 /*
 674  * Returns a mask of interrupt flags to be enabled after the whole
 675  * request has been prepared.
 676  */
 677 static u32 atmci_submit_data(struct atmel_mci *host, struct mmc_data *data)
 678 {
 679         u32 iflags;
 680
 681         data->error = -EINPROGRESS;
 682
 683         WARN_ON(host->data);
 684         host->sg = NULL;
 685         host->data = data;
 686
 687         iflags = ATMCI_DATA_ERROR_FLAGS;
 688         if (atmci_submit_data_dma(host, data)) {
 689                 host->data_chan = NULL;
 690
 691                 /*
 692                  * Errata: MMC data write operation with less than 12
 693                  * bytes is impossible.
 694                  *
 695                  * Errata: MCI Transmit Data Register (TDR) FIFO
 696                  * corruption when length is not multiple of 4.
 697                  */
 698                 if (data->blocks * data->blksz < 12
 699                                 || (data->blocks * data->blksz) & 3)
 700                         host->need_reset = true;
 701
 702                 host->sg = data->sg;
 703                 host->pio_offset = 0;
 704                 if (data->flags & MMC_DATA_READ)
 705                         iflags |= MCI_RXRDY;
 706                 else
 707                         iflags |= MCI_TXRDY;
 708         }
 709
 710         return iflags;
 711 }
 712
 713 static void atmci_start_request(struct atmel_mci *host,
 714                 struct atmel_mci_slot *slot)
 715 {
 716         struct mmc_request      *mrq;
 717         struct mmc_command      *cmd;
 718         struct mmc_data         *data;
 719         u32                     iflags;
 720         u32                     cmdflags;
 721
 722         mrq = slot->mrq;
 723         host->cur_slot = slot;
 724         host->mrq = mrq;
 725
 726         host->pending_events = 0;
 727         host->completed_events = 0;
 728         host->data_status = 0;
 729
 730         if (host->need_reset) {
 731                 mci_writel(host, CR, MCI_CR_SWRST);
 732                 mci_writel(host, CR, MCI_CR_MCIEN);
 733                 mci_writel(host, MR, host->mode_reg);
 734                 host->need_reset = false;
 735         }
 736         mci_writel(host, SDCR, slot->sdc_reg);
 737
 738         iflags = mci_readl(host, IMR);
 739         if (iflags)
 740                 dev_warn(&slot->mmc->class_dev, "WARNING: IMR=0x%08x\n",
 741                                 iflags);
 742
 743         if (unlikely(test_and_clear_bit(ATMCI_CARD_NEED_INIT, &slot->flags))) {
 744                 /* Send init sequence (74 clock cycles) */
 745                 mci_writel(host, CMDR, MCI_CMDR_SPCMD_INIT);
 746                 while (!(mci_readl(host, SR) & MCI_CMDRDY))
 747                         cpu_relax();
 748         }
 749         data = mrq->data;
 750         if (data) {
 751                 atmci_set_timeout(host, slot, data);
 752
 753                 /* Must set block count/size before sending command */
 754                 mci_writel(host, BLKR, MCI_BCNT(data->blocks)
 755                                 | MCI_BLKLEN(data->blksz));
 756                 dev_vdbg(&slot->mmc->class_dev, "BLKR=0x%08x\n",
 757                         MCI_BCNT(data->blocks) | MCI_BLKLEN(data->blksz));
 758         }
 759
 760         iflags = MCI_CMDRDY;
 761         cmd = mrq->cmd;
 762         cmdflags = atmci_prepare_command(slot->mmc, cmd);
 763         atmci_start_command(host, cmd, cmdflags);
 764
 765         if (data)
 766                 iflags |= atmci_submit_data(host, data);
 767
 768         if (mrq->stop) {
 769                 host->stop_cmdr = atmci_prepare_command(slot->mmc, mrq->stop);
 770                 host->stop_cmdr |= MCI_CMDR_STOP_XFER;
 771                 if (!(data->flags & MMC_DATA_WRITE))
 772                         host->stop_cmdr |= MCI_CMDR_TRDIR_READ;
 773                 if (data->flags & MMC_DATA_STREAM)
 774                         host->stop_cmdr |= MCI_CMDR_STREAM;
 775                 else
 776                         host->stop_cmdr |= MCI_CMDR_MULTI_BLOCK;
 777         }
 778
 779         /*
 780          * We could have enabled interrupts earlier, but I suspect
 781          * that would open up a nice can of interesting race
 782          * conditions (e.g. command and data complete, but stop not
 783          * prepared yet.)
 784          */
 785         mci_writel(host, IER, iflags);
 786 }
 787
 788 static void atmci_queue_request(struct atmel_mci *host,
 789                 struct atmel_mci_slot *slot, struct mmc_request *mrq)
 790 {
 791         dev_vdbg(&slot->mmc->class_dev, "queue request: state=%d\n",
 792                         host->state);
 793
 794         spin_lock_bh(&host->lock);
 795         slot->mrq = mrq;
 796         if (host->state == STATE_IDLE) {
 797                 host->state = STATE_SENDING_CMD;
 798                 atmci_start_request(host, slot);
 799         } else {
 800                 list_add_tail(&slot->queue_node, &host->queue);
 801         }
 802         spin_unlock_bh(&host->lock);
 803 }
 804
 805 static void atmci_request(struct mmc_host *mmc, struct mmc_request *mrq)
 806 {
 807         struct atmel_mci_slot   *slot = mmc_priv(mmc);
 808         struct atmel_mci        *host = slot->host;
 809         struct mmc_data         *data;
 810
 811         WARN_ON(slot->mrq);
 812
 813         /*
 814          * We may "know" the card is gone even though there's still an
 815          * electrical connection. If so, we really need to communicate
 816          * this to the MMC core since there won't be any more
 817          * interrupts as the card is completely removed. Otherwise,
 818          * the MMC core might believe the card is still there even
 819          * though the card was just removed very slowly.
 820          */
 821         if (!test_bit(ATMCI_CARD_PRESENT, &slot->flags)) {
 822                 mrq->cmd->error = -ENOMEDIUM;
 823                 mmc_request_done(mmc, mrq);
 824                 return;
 825         }
 826
 827         /* We don't support multiple blocks of weird lengths. */
 828         data = mrq->data;
 829         if (data && data->blocks > 1 && data->blksz & 3) {
 830                 mrq->cmd->error = -EINVAL;
 831                 mmc_request_done(mmc, mrq);
 832         }
 833
 834         atmci_queue_request(host, slot, mrq);
 835 }
 836
 837 static void atmci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
 838 {
 839         struct atmel_mci_slot   *slot = mmc_priv(mmc);
 840         struct atmel_mci        *host = slot->host;
 841         unsigned int            i;
 842
 843         slot->sdc_reg &= ~MCI_SDCBUS_MASK;
 844         switch (ios->bus_width) {
 845         case MMC_BUS_WIDTH_1:
 846                 slot->sdc_reg |= MCI_SDCBUS_1BIT;
 847                 break;
 848         case MMC_BUS_WIDTH_4:
 849                 slot->sdc_reg |= MCI_SDCBUS_4BIT;
 850                 break;
 851         }
 852
 853         if (ios->clock) {
 854                 unsigned int clock_min = ~0U;
 855                 u32 clkdiv;
 856
 857                 spin_lock_bh(&host->lock);
 858                 if (!host->mode_reg) {
 859                         clk_enable(host->mck);
 860                         mci_writel(host, CR, MCI_CR_SWRST);
 861                         mci_writel(host, CR, MCI_CR_MCIEN);
 862                 }
 863
 864                 /*
 865                  * Use mirror of ios->clock to prevent race with mmc
 866                  * core ios update when finding the minimum.
 867                  */
 868                 slot->clock = ios->clock;
 869                 for (i = 0; i < ATMEL_MCI_MAX_NR_SLOTS; i++) {
 870                         if (host->slot[i] && host->slot[i]->clock
 871                                         && host->slot[i]->clock < clock_min)
 872                                 clock_min = host->slot[i]->clock;
 873                 }
 874
 875                 /* Calculate clock divider */
 876                 clkdiv = DIV_ROUND_UP(host->bus_hz, 2 * clock_min) - 1;
 877                 if (clkdiv > 255) {
 878                         dev_warn(&mmc->class_dev,
 879                                 "clock %u too slow; using %lu\n",
 880                                 clock_min, host->bus_hz / (2 * 256));
 881                         clkdiv = 255;
 882                 }
 883
 884                 host->mode_reg = MCI_MR_CLKDIV(clkdiv);
 885
 886                 /*
 887                  * WRPROOF and RDPROOF prevent overruns/underruns by
 888                  * stopping the clock when the FIFO is full/empty.
 889                  * This state is not expected to last for long.
 890                  */
 891                 if (mci_has_rwproof())
 892                         host->mode_reg |= (MCI_MR_WRPROOF | MCI_MR_RDPROOF);
 893
 894                 if (list_empty(&host->queue))
 895                         mci_writel(host, MR, host->mode_reg);
 896                 else
 897                         host->need_clock_update = true;
 898
 899                 spin_unlock_bh(&host->lock);
 900         } else {
 901                 bool any_slot_active = false;
 902
 903                 spin_lock_bh(&host->lock);
 904                 slot->clock = 0;
 905                 for (i = 0; i < ATMEL_MCI_MAX_NR_SLOTS; i++) {
 906                         if (host->slot[i] && host->slot[i]->clock) {
 907                                 any_slot_active = true;
 908                                 break;
 909                         }
 910                 }
 911                 if (!any_slot_active) {
 912                         mci_writel(host, CR, MCI_CR_MCIDIS);
 913                         if (host->mode_reg) {
 914                                 mci_readl(host, MR);
 915                                 clk_disable(host->mck);
 916                         }
 917                         host->mode_reg = 0;
 918                 }
 919                 spin_unlock_bh(&host->lock);
 920         }
 921
 922         switch (ios->power_mode) {
 923         case MMC_POWER_UP:
 924                 set_bit(ATMCI_CARD_NEED_INIT, &slot->flags);
 925                 break;
 926         default:
 927                 /*
 928                  * TODO: None of the currently available AVR32-based
 929                  * boards allow MMC power to be turned off. Implement
 930                  * power control when this can be tested properly.
 931                  *
 932                  * We also need to hook this into the clock management
 933                  * somehow so that newly inserted cards aren't
 934                  * subjected to a fast clock before we have a chance
 935                  * to figure out what the maximum rate is. Currently,
 936                  * there's no way to avoid this, and there never will
 937                  * be for boards that don't support power control.
 938                  */
 939                 break;
 940         }
 941 }
 942
 943 static int atmci_get_ro(struct mmc_host *mmc)
 944 {
 945         int                     read_only = -ENOSYS;
 946         struct atmel_mci_slot   *slot = mmc_priv(mmc);
 947
 948         if (gpio_is_valid(slot->wp_pin)) {
 949                 read_only = gpio_get_value(slot->wp_pin);
 950                 dev_dbg(&mmc->class_dev, "card is %s\n",
 951                                 read_only ? "read-only" : "read-write");
 952         }
 953
 954         return read_only;
 955 }
 956
 957 static int atmci_get_cd(struct mmc_host *mmc)
 958 {
 959         int                     present = -ENOSYS;
 960         struct atmel_mci_slot   *slot = mmc_priv(mmc);
 961
 962         if (gpio_is_valid(slot->detect_pin)) {
 963                 present = !(gpio_get_value(slot->detect_pin) ^
 964                             slot->detect_is_active_high);
 965                 dev_dbg(&mmc->class_dev, "card is %spresent\n",
 966                                 present ? "" : "not ");
 967         }
 968
 969         return present;
 970 }
 971
 972 static const struct mmc_host_ops atmci_ops = {
 973         .request        = atmci_request,
 974         .set_ios        = atmci_set_ios,
 975         .get_ro         = atmci_get_ro,
 976         .get_cd         = atmci_get_cd,
 977 };
 978
 979 /* Called with host->lock held */
 980 static void atmci_request_end(struct atmel_mci *host, struct mmc_request *mrq)
 981         __releases(&host->lock)
 982         __acquires(&host->lock)
 983 {
 984         struct atmel_mci_slot   *slot = NULL;
 985         struct mmc_host         *prev_mmc = host->cur_slot->mmc;
 986
 987         WARN_ON(host->cmd || host->data);
 988
 989         /*
 990          * Update the MMC clock rate if necessary. This may be
 991          * necessary if set_ios() is called when a different slot is
 992          * busy transfering data.
 993          */
 994         if (host->need_clock_update)
 995                 mci_writel(host, MR, host->mode_reg);
 996
 997         host->cur_slot->mrq = NULL;
 998         host->mrq = NULL;
 999         if (!list_empty(&host->queue)) {
1000                 slot = list_entry(host->queue.next,
1001                                 struct atmel_mci_slot, queue_node);
1002                 list_del(&slot->queue_node);
1003                 dev_vdbg(&host->pdev->dev, "list not empty: %s is next\n",
1004                                 mmc_hostname(slot->mmc));
1005                 host->state = STATE_SENDING_CMD;
1006                 atmci_start_request(host, slot);
1007         } else {
1008                 dev_vdbg(&host->pdev->dev, "list empty\n");
1009                 host->state = STATE_IDLE;
1010         }
1011
1012         spin_unlock(&host->lock);
1013         mmc_request_done(prev_mmc, mrq);
1014         spin_lock(&host->lock);
1015 }
1016
1017 static void atmci_command_complete(struct atmel_mci *host,
1018                         struct mmc_command *cmd)
1019 {
1020         u32             status = host->cmd_status;
1021
1022         /* Read the response from the card (up to 16 bytes) */
1023         cmd->resp[0] = mci_readl(host, RSPR);
1024         cmd->resp[1] = mci_readl(host, RSPR);
1025         cmd->resp[2] = mci_readl(host, RSPR);
1026         cmd->resp[3] = mci_readl(host, RSPR);
1027
1028         if (status & MCI_RTOE)
1029                 cmd->error = -ETIMEDOUT;
1030         else if ((cmd->flags & MMC_RSP_CRC) && (status & MCI_RCRCE))
1031                 cmd->error = -EILSEQ;
1032         else if (status & (MCI_RINDE | MCI_RDIRE | MCI_RENDE))
1033                 cmd->error = -EIO;
1034         else
1035                 cmd->error = 0;
1036
1037         if (cmd->error) {
1038                 dev_dbg(&host->pdev->dev,
1039                         "command error: status=0x%08x\n", status);
1040
1041                 if (cmd->data) {
1042                         host->data = NULL;
1043                         atmci_stop_dma(host);
1044                         mci_writel(host, IDR, MCI_NOTBUSY
1045                                         | MCI_TXRDY | MCI_RXRDY
1046                                         | ATMCI_DATA_ERROR_FLAGS);
1047                 }
1048         }
1049 }
1050
1051 static void atmci_detect_change(unsigned long data)
1052 {
1053         struct atmel_mci_slot   *slot = (struct atmel_mci_slot *)data;
1054         bool                    present;
1055         bool                    present_old;
1056
1057         /*
1058          * atmci_cleanup_slot() sets the ATMCI_SHUTDOWN flag before
1059          * freeing the interrupt. We must not re-enable the interrupt
1060          * if it has been freed, and if we're shutting down, it
1061          * doesn't really matter whether the card is present or not.
1062          */
1063         smp_rmb();
1064         if (test_bit(ATMCI_SHUTDOWN, &slot->flags))
1065                 return;
1066
1067         enable_irq(gpio_to_irq(slot->detect_pin));
1068         present = !(gpio_get_value(slot->detect_pin) ^
1069                     slot->detect_is_active_high);
1070         present_old = test_bit(ATMCI_CARD_PRESENT, &slot->flags);
1071
1072         dev_vdbg(&slot->mmc->class_dev, "detect change: %d (was %d)\n",
1073                         present, present_old);
1074
1075         if (present != present_old) {
1076                 struct atmel_mci        *host = slot->host;
1077                 struct mmc_request      *mrq;
1078
1079                 dev_dbg(&slot->mmc->class_dev, "card %s\n",
1080                         present ? "inserted" : "removed");
1081
1082                 spin_lock(&host->lock);
1083
1084                 if (!present)
1085                         clear_bit(ATMCI_CARD_PRESENT, &slot->flags);
1086                 else
1087                         set_bit(ATMCI_CARD_PRESENT, &slot->flags);
1088
1089                 /* Clean up queue if present */
1090                 mrq = slot->mrq;
1091                 if (mrq) {
1092                         if (mrq == host->mrq) {
1093                                 /*
1094                                  * Reset controller to terminate any ongoing
1095                                  * commands or data transfers.
1096                                  */
1097                                 mci_writel(host, CR, MCI_CR_SWRST);
1098                                 mci_writel(host, CR, MCI_CR_MCIEN);
1099                                 mci_writel(host, MR, host->mode_reg);
1100
1101                                 host->data = NULL;
1102                                 host->cmd = NULL;
1103
1104                                 switch (host->state) {
1105                                 case STATE_IDLE:
1106                                         break;
1107                                 case STATE_SENDING_CMD:
1108                                         mrq->cmd->error = -ENOMEDIUM;
1109                                         if (!mrq->data)
1110                                                 break;
1111                                         /* fall through */
1112                                 case STATE_SENDING_DATA:
1113                                         mrq->data->error = -ENOMEDIUM;
1114                                         atmci_stop_dma(host);
1115                                         break;
1116                                 case STATE_DATA_BUSY:
1117                                 case STATE_DATA_ERROR:
1118                                         if (mrq->data->error == -EINPROGRESS)
1119                                                 mrq->data->error = -ENOMEDIUM;
1120                                         if (!mrq->stop)
1121                                                 break;
1122                                         /* fall through */
1123                                 case STATE_SENDING_STOP:
1124                                         mrq->stop->error = -ENOMEDIUM;
1125                                         break;
1126                                 }
1127
1128                                 atmci_request_end(host, mrq);
1129                         } else {
1130                                 list_del(&slot->queue_node);
1131                                 mrq->cmd->error = -ENOMEDIUM;
1132                                 if (mrq->data)
1133                                         mrq->data->error = -ENOMEDIUM;
1134                                 if (mrq->stop)
1135                                         mrq->stop->error = -ENOMEDIUM;
1136
1137                                 spin_unlock(&host->lock);
1138                                 mmc_request_done(slot->mmc, mrq);
1139                                 spin_lock(&host->lock);
1140                         }
1141                 }
1142                 spin_unlock(&host->lock);
1143
1144                 mmc_detect_change(slot->mmc, 0);
1145         }
1146 }
1147
1148 static void atmci_tasklet_func(unsigned long priv)
1149 {
1150         struct atmel_mci        *host = (struct atmel_mci *)priv;
1151         struct mmc_request      *mrq = host->mrq;
1152         struct mmc_data         *data = host->data;
1153         struct mmc_command      *cmd = host->cmd;
1154         enum atmel_mci_state    state = host->state;
1155         enum atmel_mci_state    prev_state;
1156         u32                     status;
1157
1158         spin_lock(&host->lock);
1159
1160         state = host->state;
1161
1162         dev_vdbg(&host->pdev->dev,
1163                 "tasklet: state %u pending/completed/mask %lx/%lx/%x\n",
1164                 state, host->pending_events, host->completed_events,
1165                 mci_readl(host, IMR));
1166
1167         do {
1168                 prev_state = state;
1169
1170                 switch (state) {
1171                 case STATE_IDLE:
1172                         break;
1173
1174                 case STATE_SENDING_CMD:
1175                         if (!atmci_test_and_clear_pending(host,
1176                                                 EVENT_CMD_COMPLETE))
1177                                 break;
1178
1179                         host->cmd = NULL;
1180                         atmci_set_completed(host, EVENT_CMD_COMPLETE);
1181                         atmci_command_complete(host, mrq->cmd);
1182                         if (!mrq->data || cmd->error) {
1183                                 atmci_request_end(host, host->mrq);
1184                                 goto unlock;
1185                         }
1186
1187                         prev_state = state = STATE_SENDING_DATA;
1188                         /* fall through */
1189
1190                 case STATE_SENDING_DATA:
1191                         if (atmci_test_and_clear_pending(host,
1192                                                 EVENT_DATA_ERROR)) {
1193                                 atmci_stop_dma(host);
1194                                 if (data->stop)
1195                                         send_stop_cmd(host, data);
1196                                 state = STATE_DATA_ERROR;
1197                                 break;
1198                         }
1199
1200                         if (!atmci_test_and_clear_pending(host,
1201                                                 EVENT_XFER_COMPLETE))
1202                                 break;
1203
1204                         atmci_set_completed(host, EVENT_XFER_COMPLETE);
1205                         prev_state = state = STATE_DATA_BUSY;
1206                         /* fall through */
1207
1208                 case STATE_DATA_BUSY:
1209                         if (!atmci_test_and_clear_pending(host,
1210                                                 EVENT_DATA_COMPLETE))
1211                                 break;
1212
1213                         host->data = NULL;
1214                         atmci_set_completed(host, EVENT_DATA_COMPLETE);
1215                         status = host->data_status;
1216                         if (unlikely(status & ATMCI_DATA_ERROR_FLAGS)) {
1217                                 if (status & MCI_DTOE) {
1218                                         dev_dbg(&host->pdev->dev,
1219                                                         "data timeout error\n");
1220                                         data->error = -ETIMEDOUT;
1221                                 } else if (status & MCI_DCRCE) {
1222                                         dev_dbg(&host->pdev->dev,
1223                                                         "data CRC error\n");
1224                                         data->error = -EILSEQ;
1225                                 } else {
1226                                         dev_dbg(&host->pdev->dev,
1227                                                 "data FIFO error (status=%08x)\n",
1228                                                 status);
1229                                         data->error = -EIO;
1230                                 }
1231                         } else {
1232                                 data->bytes_xfered = data->blocks * data->blksz;
1233                                 data->error = 0;
1234                         }
1235
1236                         if (!data->stop) {
1237                                 atmci_request_end(host, host->mrq);
1238                                 goto unlock;
1239                         }
1240
1241                         prev_state = state = STATE_SENDING_STOP;
1242                         if (!data->error)
1243                                 send_stop_cmd(host, data);
1244                         /* fall through */
1245
1246                 case STATE_SENDING_STOP:
1247                         if (!atmci_test_and_clear_pending(host,
1248                                                 EVENT_CMD_COMPLETE))
1249                                 break;
1250
1251                         host->cmd = NULL;
1252                         atmci_command_complete(host, mrq->stop);
1253                         atmci_request_end(host, host->mrq);
1254                         goto unlock;
1255
1256                 case STATE_DATA_ERROR:
1257                         if (!atmci_test_and_clear_pending(host,
1258                                                 EVENT_XFER_COMPLETE))
1259                                 break;
1260
1261                         state = STATE_DATA_BUSY;
1262                         break;
1263                 }
1264         } while (state != prev_state);
1265
1266         host->state = state;
1267
1268 unlock:
1269         spin_unlock(&host->lock);
1270 }
1271
1272 static void atmci_read_data_pio(struct atmel_mci *host)
1273 {
1274         struct scatterlist      *sg = host->sg;
1275         void                    *buf = sg_virt(sg);
1276         unsigned int            offset = host->pio_offset;
1277         struct mmc_data         *data = host->data;
1278         u32                     value;
1279         u32                     status;
1280         unsigned int            nbytes = 0;
1281
1282         do {
1283                 value = mci_readl(host, RDR);
1284                 if (likely(offset + 4 <= sg->length)) {
1285                         put_unaligned(value, (u32 *)(buf + offset));
1286
1287                         offset += 4;
1288                         nbytes += 4;
1289
1290                         if (offset == sg->length) {
1291                                 flush_dcache_page(sg_page(sg));
1292                                 host->sg = sg = sg_next(sg);
1293                                 if (!sg)
1294                                         goto done;
1295
1296                                 offset = 0;
1297                                 buf = sg_virt(sg);
1298                         }
1299                 } else {
1300                         unsigned int remaining = sg->length - offset;
1301                         memcpy(buf + offset, &value, remaining);
1302                         nbytes += remaining;
1303
1304                         flush_dcache_page(sg_page(sg));
1305                         host->sg = sg = sg_next(sg);
1306                         if (!sg)
1307                                 goto done;
1308
1309                         offset = 4 - remaining;
1310                         buf = sg_virt(sg);
1311                         memcpy(buf, (u8 *)&value + remaining, offset);
1312                         nbytes += offset;
1313                 }
1314
1315                 status = mci_readl(host, SR);
1316                 if (status & ATMCI_DATA_ERROR_FLAGS) {
1317                         mci_writel(host, IDR, (MCI_NOTBUSY | MCI_RXRDY
1318                                                 | ATMCI_DATA_ERROR_FLAGS));
1319                         host->data_status = status;
1320                         data->bytes_xfered += nbytes;
1321                         smp_wmb();
1322                         atmci_set_pending(host, EVENT_DATA_ERROR);
1323                         tasklet_schedule(&host->tasklet);
1324                         return;
1325                 }
1326         } while (status & MCI_RXRDY);
1327
1328         host->pio_offset = offset;
1329         data->bytes_xfered += nbytes;
1330
1331         return;
1332
1333 done:
1334         mci_writel(host, IDR, MCI_RXRDY);
1335         mci_writel(host, IER, MCI_NOTBUSY);
1336         data->bytes_xfered += nbytes;
1337         smp_wmb();
1338         atmci_set_pending(host, EVENT_XFER_COMPLETE);
1339 }
1340
1341 static void atmci_write_data_pio(struct atmel_mci *host)
1342 {
1343         struct scatterlist      *sg = host->sg;
1344         void                    *buf = sg_virt(sg);
1345         unsigned int            offset = host->pio_offset;
1346         struct mmc_data         *data = host->data;
1347         u32                     value;
1348         u32                     status;
1349         unsigned int            nbytes = 0;
1350
1351         do {
1352                 if (likely(offset + 4 <= sg->length)) {
1353                         value = get_unaligned((u32 *)(buf + offset));
1354                         mci_writel(host, TDR, value);
1355
1356                         offset += 4;
1357                         nbytes += 4;
1358                         if (offset == sg->length) {
1359                                 host->sg = sg = sg_next(sg);
1360                                 if (!sg)
1361                                         goto done;
1362
1363                                 offset = 0;
1364                                 buf = sg_virt(sg);
1365                         }
1366                 } else {
1367                         unsigned int remaining = sg->length - offset;
1368
1369                         value = 0;
1370                         memcpy(&value, buf + offset, remaining);
1371                         nbytes += remaining;
1372
1373                         host->sg = sg = sg_next(sg);
1374                         if (!sg) {
1375                                 mci_writel(host, TDR, value);
1376                                 goto done;
1377                         }
1378
1379                         offset = 4 - remaining;
1380                         buf = sg_virt(sg);
1381                         memcpy((u8 *)&value + remaining, buf, offset);
1382                         mci_writel(host, TDR, value);
1383                         nbytes += offset;
1384                 }
1385
1386                 status = mci_readl(host, SR);
1387                 if (status & ATMCI_DATA_ERROR_FLAGS) {
1388                         mci_writel(host, IDR, (MCI_NOTBUSY | MCI_TXRDY
1389                                                 | ATMCI_DATA_ERROR_FLAGS));
1390                         host->data_status = status;
1391                         data->bytes_xfered += nbytes;
1392                         smp_wmb();
1393                         atmci_set_pending(host, EVENT_DATA_ERROR);
1394                         tasklet_schedule(&host->tasklet);
1395                         return;
1396                 }
1397         } while (status & MCI_TXRDY);
1398
1399         host->pio_offset = offset;
1400         data->bytes_xfered += nbytes;
1401
1402         return;
1403
1404 done:
1405         mci_writel(host, IDR, MCI_TXRDY);
1406         mci_writel(host, IER, MCI_NOTBUSY);
1407         data->bytes_xfered += nbytes;
1408         smp_wmb();
1409         atmci_set_pending(host, EVENT_XFER_COMPLETE);
1410 }
1411
1412 static void atmci_cmd_interrupt(struct atmel_mci *host, u32 status)
1413 {
1414         mci_writel(host, IDR, MCI_CMDRDY);
1415
1416         host->cmd_status = status;
1417         smp_wmb();
1418         atmci_set_pending(host, EVENT_CMD_COMPLETE);
1419         tasklet_schedule(&host->tasklet);
1420 }
1421
1422 static irqreturn_t atmci_interrupt(int irq, void *dev_id)
1423 {
1424         struct atmel_mci        *host = dev_id;
1425         u32                     status, mask, pending;
1426         unsigned int            pass_count = 0;
1427
1428         do {
1429                 status = mci_readl(host, SR);
1430                 mask = mci_readl(host, IMR);
1431                 pending = status & mask;
1432                 if (!pending)
1433                         break;
1434
1435                 if (pending & ATMCI_DATA_ERROR_FLAGS) {
1436                         mci_writel(host, IDR, ATMCI_DATA_ERROR_FLAGS
1437                                         | MCI_RXRDY | MCI_TXRDY);
1438                         pending &= mci_readl(host, IMR);
1439
1440                         host->data_status = status;
1441                         smp_wmb();
1442                         atmci_set_pending(host, EVENT_DATA_ERROR);
1443                         tasklet_schedule(&host->tasklet);
1444                 }
1445                 if (pending & MCI_NOTBUSY) {
1446                         mci_writel(host, IDR,
1447                                         ATMCI_DATA_ERROR_FLAGS | MCI_NOTBUSY);
1448                         if (!host->data_status)
1449                                 host->data_status = status;
1450                         smp_wmb();
1451                         atmci_set_pending(host, EVENT_DATA_COMPLETE);
1452                         tasklet_schedule(&host->tasklet);
1453                 }
1454                 if (pending & MCI_RXRDY)
1455                         atmci_read_data_pio(host);
1456                 if (pending & MCI_TXRDY)
1457                         atmci_write_data_pio(host);
1458
1459                 if (pending & MCI_CMDRDY)
1460                         atmci_cmd_interrupt(host, status);
1461         } while (pass_count++ < 5);
1462
1463         return pass_count ? IRQ_HANDLED : IRQ_NONE;
1464 }
1465
1466 static irqreturn_t atmci_detect_interrupt(int irq, void *dev_id)
1467 {
1468         struct atmel_mci_slot   *slot = dev_id;
1469
1470         /*
1471          * Disable interrupts until the pin has stabilized and check
1472          * the state then. Use mod_timer() since we may be in the
1473          * middle of the timer routine when this interrupt triggers.
1474          */
1475         disable_irq_nosync(irq);
1476         mod_timer(&slot->detect_timer, jiffies + msecs_to_jiffies(20));
1477
1478         return IRQ_HANDLED;
1479 }
1480
1481 static int __init atmci_init_slot(struct atmel_mci *host,
1482                 struct mci_slot_pdata *slot_data, unsigned int id,
1483                 u32 sdc_reg)
1484 {
1485         struct mmc_host                 *mmc;
1486         struct atmel_mci_slot           *slot;
1487
1488         mmc = mmc_alloc_host(sizeof(struct atmel_mci_slot), &host->pdev->dev);
1489         if (!mmc)
1490                 return -ENOMEM;
1491
1492         slot = mmc_priv(mmc);
1493         slot->mmc = mmc;
1494         slot->host = host;
1495         slot->detect_pin = slot_data->detect_pin;
1496         slot->wp_pin = slot_data->wp_pin;
1497         slot->detect_is_active_high = slot_data->detect_is_active_high;
1498         slot->sdc_reg = sdc_reg;
1499
1500         mmc->ops = &atmci_ops;
1501         mmc->f_min = DIV_ROUND_UP(host->bus_hz, 512);
1502         mmc->f_max = host->bus_hz / 2;
1503         mmc->ocr_avail  = MMC_VDD_32_33 | MMC_VDD_33_34;
1504         if (slot_data->bus_width >= 4)
1505                 mmc->caps |= MMC_CAP_4_BIT_DATA;
1506
1507         mmc->max_hw_segs = 64;
1508         mmc->max_phys_segs = 64;
1509         mmc->max_req_size = 32768 * 512;
1510         mmc->max_blk_size = 32768;
1511         mmc->max_blk_count = 512;
1512
1513         /* Assume card is present initially */
1514         set_bit(ATMCI_CARD_PRESENT, &slot->flags);
1515         if (gpio_is_valid(slot->detect_pin)) {
1516                 if (gpio_request(slot->detect_pin, "mmc_detect")) {
1517                         dev_dbg(&mmc->class_dev, "no detect pin available\n");
1518                         slot->detect_pin = -EBUSY;
1519                 } else if (gpio_get_value(slot->detect_pin) ^
1520                                 slot->detect_is_active_high) {
1521                         clear_bit(ATMCI_CARD_PRESENT, &slot->flags);
1522                 }
1523         }
1524
1525         if (!gpio_is_valid(slot->detect_pin))
1526                 mmc->caps |= MMC_CAP_NEEDS_POLL;
1527
1528         if (gpio_is_valid(slot->wp_pin)) {
1529                 if (gpio_request(slot->wp_pin, "mmc_wp")) {
1530                         dev_dbg(&mmc->class_dev, "no WP pin available\n");
1531                         slot->wp_pin = -EBUSY;
1532                 }
1533         }
1534
1535         host->slot[id] = slot;
1536         mmc_add_host(mmc);
1537
1538         if (gpio_is_valid(slot->detect_pin)) {
1539                 int ret;
1540
1541                 setup_timer(&slot->detect_timer, atmci_detect_change,
1542                                 (unsigned long)slot);
1543
1544                 ret = request_irq(gpio_to_irq(slot->detect_pin),
1545                                 atmci_detect_interrupt,
1546                                 IRQF_TRIGGER_FALLING | IRQF_TRIGGER_RISING,
1547                                 "mmc-detect", slot);
1548                 if (ret) {
1549                         dev_dbg(&mmc->class_dev,
1550                                 "could not request IRQ %d for detect pin\n",
1551                                 gpio_to_irq(slot->detect_pin));
1552                         gpio_free(slot->detect_pin);
1553                         slot->detect_pin = -EBUSY;
1554                 }
1555         }
1556
1557         atmci_init_debugfs(slot);
1558
1559         return 0;
1560 }
1561
1562 static void __exit atmci_cleanup_slot(struct atmel_mci_slot *slot,
1563                 unsigned int id)
1564 {
1565         /* Debugfs stuff is cleaned up by mmc core */
1566
1567         set_bit(ATMCI_SHUTDOWN, &slot->flags);
1568         smp_wmb();
1569
1570         mmc_remove_host(slot->mmc);
1571
1572         if (gpio_is_valid(slot->detect_pin)) {
1573                 int pin = slot->detect_pin;
1574
1575                 free_irq(gpio_to_irq(pin), slot);
1576                 del_timer_sync(&slot->detect_timer);
1577                 gpio_free(pin);
1578         }
1579         if (gpio_is_valid(slot->wp_pin))
1580                 gpio_free(slot->wp_pin);
1581
1582         slot->host->slot[id] = NULL;
1583         mmc_free_host(slot->mmc);
1584 }
1585
1586 #ifdef CONFIG_MMC_ATMELMCI_DMA
1587 static bool filter(struct dma_chan *chan, void *slave)
1588 {
1589         struct mci_dma_data     *sl = slave;
1590
1591         if (sl && find_slave_dev(sl) == chan->device->dev) {
1592                 chan->private = slave_data_ptr(sl);
1593                 return true;
1594         } else {
1595                 return false;
1596         }
1597 }
1598
1599 static void atmci_configure_dma(struct atmel_mci *host)
1600 {
1601         struct mci_platform_data        *pdata;
1602
1603         if (host == NULL)
1604                 return;
1605
1606         pdata = host->pdev->dev.platform_data;
1607
1608         if (pdata && find_slave_dev(pdata->dma_slave)) {
1609                 dma_cap_mask_t mask;
1610
1611                 setup_dma_addr(pdata->dma_slave,
1612                                host->mapbase + MCI_TDR,
1613                                host->mapbase + MCI_RDR);
1614
1615                 /* Try to grab a DMA channel */
1616                 dma_cap_zero(mask);
1617                 dma_cap_set(DMA_SLAVE, mask);
1618                 host->dma.chan =
1619                         dma_request_channel(mask, filter, pdata->dma_slave);
1620         }
1621         if (!host->dma.chan)
1622                 dev_notice(&host->pdev->dev, "DMA not available, using PIO\n");
1623 }
1624 #else
1625 static void atmci_configure_dma(struct atmel_mci *host) {}
1626 #endif
1627
1628 static int __init atmci_probe(struct platform_device *pdev)
1629 {
1630         struct mci_platform_data        *pdata;
1631         struct atmel_mci                *host;
1632         struct resource                 *regs;
1633         unsigned int                    nr_slots;
1634         int                             irq;
1635         int                             ret;
1636
1637         regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1638         if (!regs)
1639                 return -ENXIO;
1640         pdata = pdev->dev.platform_data;
1641         if (!pdata)
1642                 return -ENXIO;
1643         irq = platform_get_irq(pdev, 0);
1644         if (irq < 0)
1645                 return irq;
1646
1647         host = kzalloc(sizeof(struct atmel_mci), GFP_KERNEL);
1648         if (!host)
1649                 return -ENOMEM;
1650
1651         host->pdev = pdev;
1652         spin_lock_init(&host->lock);
1653         INIT_LIST_HEAD(&host->queue);
1654
1655         host->mck = clk_get(&pdev->dev, "mci_clk");
1656         if (IS_ERR(host->mck)) {
1657                 ret = PTR_ERR(host->mck);
1658                 goto err_clk_get;
1659         }
1660
1661         ret = -ENOMEM;
1662         host->regs = ioremap(regs->start, regs->end - regs->start + 1);
1663         if (!host->regs)
1664                 goto err_ioremap;
1665
1666         clk_enable(host->mck);
1667         mci_writel(host, CR, MCI_CR_SWRST);
1668         host->bus_hz = clk_get_rate(host->mck);
1669         clk_disable(host->mck);
1670
1671         host->mapbase = regs->start;
1672
1673         tasklet_init(&host->tasklet, atmci_tasklet_func, (unsigned long)host);
1674
1675         ret = request_irq(irq, atmci_interrupt, 0, dev_name(&pdev->dev), host);
1676         if (ret)
1677                 goto err_request_irq;
1678
1679         atmci_configure_dma(host);
1680
1681         platform_set_drvdata(pdev, host);
1682
1683         /* We need at least one slot to succeed */
1684         nr_slots = 0;
1685         ret = -ENODEV;
1686         if (pdata->slot[0].bus_width) {
1687                 ret = atmci_init_slot(host, &pdata->slot[0],
1688                                 MCI_SDCSEL_SLOT_A, 0);
1689                 if (!ret)
1690                         nr_slots++;
1691         }
1692         if (pdata->slot[1].bus_width) {
1693                 ret = atmci_init_slot(host, &pdata->slot[1],
1694                                 MCI_SDCSEL_SLOT_B, 1);
1695                 if (!ret)
1696                         nr_slots++;
1697         }
1698
1699         if (!nr_slots) {
1700                 dev_err(&pdev->dev, "init failed: no slot defined\n");
1701                 goto err_init_slot;
1702         }
1703
1704         dev_info(&pdev->dev,
1705                         "Atmel MCI controller at 0x%08lx irq %d, %u slots\n",
1706                         host->mapbase, irq, nr_slots);
1707
1708         return 0;
1709
1710 err_init_slot:
1711 #ifdef CONFIG_MMC_ATMELMCI_DMA
1712         if (host->dma.chan)
1713                 dma_release_channel(host->dma.chan);
1714 #endif
1715         free_irq(irq, host);
1716 err_request_irq:
1717         iounmap(host->regs);
1718 err_ioremap:
1719         clk_put(host->mck);
1720 err_clk_get:
1721         kfree(host);
1722         return ret;
1723 }
1724
1725 static int __exit atmci_remove(struct platform_device *pdev)
1726 {
1727         struct atmel_mci        *host = platform_get_drvdata(pdev);
1728         unsigned int            i;
1729
1730         platform_set_drvdata(pdev, NULL);
1731
1732         for (i = 0; i < ATMEL_MCI_MAX_NR_SLOTS; i++) {
1733                 if (host->slot[i])
1734                         atmci_cleanup_slot(host->slot[i], i);
1735         }
1736
1737         clk_enable(host->mck);
1738         mci_writel(host, IDR, ~0UL);
1739         mci_writel(host, CR, MCI_CR_MCIDIS);
1740         mci_readl(host, SR);
1741         clk_disable(host->mck);
1742
1743 #ifdef CONFIG_MMC_ATMELMCI_DMA
1744         if (host->dma.chan)
1745                 dma_release_channel(host->dma.chan);
1746 #endif
1747
1748         free_irq(platform_get_irq(pdev, 0), host);
1749         iounmap(host->regs);
1750
1751         clk_put(host->mck);
1752         kfree(host);
1753
1754         return 0;
1755 }
1756
1757 static struct platform_driver atmci_driver = {
1758         .remove         = __exit_p(atmci_remove),
1759         .driver         = {
1760                 .name           = "atmel_mci",
1761         },
1762 };
1763
1764 static int __init atmci_init(void)
1765 {
1766         return platform_driver_probe(&atmci_driver, atmci_probe);
1767 }
1768
1769 static void __exit atmci_exit(void)
1770 {
1771         platform_driver_unregister(&atmci_driver);
1772 }
1773
1774 late_initcall(atmci_init); /* try to load after dma driver when built-in */
1775 module_exit(atmci_exit);
1776
1777 MODULE_DESCRIPTION("Atmel Multimedia Card Interface driver");
1778 MODULE_AUTHOR("Haavard Skinnemoen <haavard.skinnemoen@atmel.com>");
1779 MODULE_LICENSE("GPL v2");