]> bbs.cooldavid.org Git - net-next-2.6.git/blame - drivers/ata/ahci.c
git://bbs.cooldavid.org / net-next-2.6.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
libata: remiplement ata_hpa_resize()
[net-next-2.6.git] / drivers / ata / ahci.c
CommitLineData
1da177e4
LT		 1/*
2 * ahci.c - AHCI SATA support
3 *
af36d7f0
JG		 4 * Maintained by: Jeff Garzik <jgarzik@pobox.com>
5 * Please ALWAYS copy linux-ide@vger.kernel.org
6 * on emails.
7 *
8 * Copyright 2004-2005 Red Hat, Inc.
9 *
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2, or (at your option)
14 * any later version.
15 *
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
20 *
21 * You should have received a copy of the GNU General Public License
22 * along with this program; see the file COPYING. If not, write to
23 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
24 *
25 *
26 * libata documentation is available via 'make {ps|pdf}docs',
27 * as Documentation/DocBook/libata.*
28 *
29 * AHCI hardware documentation:
1da177e4 30 * http://www.intel.com/technology/serialata/pdf/rev1_0.pdf
af36d7f0 31 * http://www.intel.com/technology/serialata/pdf/rev1_1.pdf
1da177e4
LT		 32 *
33 */
34
35#include <linux/kernel.h>
36#include <linux/module.h>
37#include <linux/pci.h>
38#include <linux/init.h>
39#include <linux/blkdev.h>
40#include <linux/delay.h>
41#include <linux/interrupt.h>
87507cfd 42#include <linux/dma-mapping.h>
a9524a76 43#include <linux/device.h>
1da177e4 44#include <scsi/scsi_host.h>
193515d5 45#include <scsi/scsi_cmnd.h>
1da177e4 46#include <linux/libata.h>
1da177e4
LT		 47
48#define DRV_NAME "ahci"
cd70c266 49#define DRV_VERSION "2.3"
1da177e4
LT		 50
51
52enum {
53 AHCI_PCI_BAR = 5,
648a88be 54 AHCI_MAX_PORTS = 32,
1da177e4
LT		 55 AHCI_MAX_SG = 168, /* hardware max is 64K */
56 AHCI_DMA_BOUNDARY = 0xffffffff,
be5d8218 57 AHCI_USE_CLUSTERING = 1,
12fad3f9 58 AHCI_MAX_CMDS = 32,
dd410ff1 59 AHCI_CMD_SZ = 32,
12fad3f9 60 AHCI_CMD_SLOT_SZ = AHCI_MAX_CMDS * AHCI_CMD_SZ,
1da177e4 61 AHCI_RX_FIS_SZ = 256,
a0ea7328 62 AHCI_CMD_TBL_CDB = 0x40,
dd410ff1
TH		 63 AHCI_CMD_TBL_HDR_SZ = 0x80,
64 AHCI_CMD_TBL_SZ = AHCI_CMD_TBL_HDR_SZ + (AHCI_MAX_SG * 16),
65 AHCI_CMD_TBL_AR_SZ = AHCI_CMD_TBL_SZ * AHCI_MAX_CMDS,
66 AHCI_PORT_PRIV_DMA_SZ = AHCI_CMD_SLOT_SZ + AHCI_CMD_TBL_AR_SZ +
1da177e4
LT		 67 AHCI_RX_FIS_SZ,
68 AHCI_IRQ_ON_SG = (1 << 31),
69 AHCI_CMD_ATAPI = (1 << 5),
70 AHCI_CMD_WRITE = (1 << 6),
4b10e559 71 AHCI_CMD_PREFETCH = (1 << 7),
22b49985
TH		 72 AHCI_CMD_RESET = (1 << 8),
73 AHCI_CMD_CLR_BUSY = (1 << 10),
1da177e4
LT		 74
75 RX_FIS_D2H_REG = 0x40, /* offset of D2H Register FIS data */
0291f95f 76 RX_FIS_SDB = 0x58, /* offset of SDB FIS data */
78cd52d0 77 RX_FIS_UNK = 0x60, /* offset of Unknown FIS data */
1da177e4
LT		 78
79 board_ahci = 0,
648a88be
TH		 80 board_ahci_pi = 1,
81 board_ahci_vt8251 = 2,
82 board_ahci_ign_iferr = 3,
55a61604 83 board_ahci_sb600 = 4,
cd70c266 84 board_ahci_mv = 5,
1da177e4
LT		 85
86 /* global controller registers */
87 HOST_CAP = 0x00, /* host capabilities */
88 HOST_CTL = 0x04, /* global host control */
89 HOST_IRQ_STAT = 0x08, /* interrupt status */
90 HOST_PORTS_IMPL = 0x0c, /* bitmap of implemented ports */
91 HOST_VERSION = 0x10, /* AHCI spec. version compliancy */
92
93 /* HOST_CTL bits */
94 HOST_RESET = (1 << 0), /* reset controller; self-clear */
95 HOST_IRQ_EN = (1 << 1), /* global IRQ enable */
96 HOST_AHCI_EN = (1 << 31), /* AHCI enabled */
97
98 /* HOST_CAP bits */
0be0aa98 99 HOST_CAP_SSC = (1 << 14), /* Slumber capable */
22b49985 100 HOST_CAP_CLO = (1 << 24), /* Command List Override support */
0be0aa98 101 HOST_CAP_SSS = (1 << 27), /* Staggered Spin-up */
203ef6c4 102 HOST_CAP_SNTF = (1 << 29), /* SNotification register */
979db803 103 HOST_CAP_NCQ = (1 << 30), /* Native Command Queueing */
dd410ff1 104 HOST_CAP_64 = (1 << 31), /* PCI DAC (64-bit DMA) support */
1da177e4
LT		 105
106 /* registers for each SATA port */
107 PORT_LST_ADDR = 0x00, /* command list DMA addr */
108 PORT_LST_ADDR_HI = 0x04, /* command list DMA addr hi */
109 PORT_FIS_ADDR = 0x08, /* FIS rx buf addr */
110 PORT_FIS_ADDR_HI = 0x0c, /* FIS rx buf addr hi */
111 PORT_IRQ_STAT = 0x10, /* interrupt status */
112 PORT_IRQ_MASK = 0x14, /* interrupt enable/disable mask */
113 PORT_CMD = 0x18, /* port command */
114 PORT_TFDATA = 0x20, /* taskfile data */
115 PORT_SIG = 0x24, /* device TF signature */
116 PORT_CMD_ISSUE = 0x38, /* command issue */
1da177e4
LT		 117 PORT_SCR_STAT = 0x28, /* SATA phy register: SStatus */
118 PORT_SCR_CTL = 0x2c, /* SATA phy register: SControl */
119 PORT_SCR_ERR = 0x30, /* SATA phy register: SError */
120 PORT_SCR_ACT = 0x34, /* SATA phy register: SActive */
203ef6c4 121 PORT_SCR_NTF = 0x3c, /* SATA phy register: SNotification */
1da177e4
LT		 122
123 /* PORT_IRQ_{STAT,MASK} bits */
124 PORT_IRQ_COLD_PRES = (1 << 31), /* cold presence detect */
125 PORT_IRQ_TF_ERR = (1 << 30), /* task file error */
126 PORT_IRQ_HBUS_ERR = (1 << 29), /* host bus fatal error */
127 PORT_IRQ_HBUS_DATA_ERR = (1 << 28), /* host bus data error */
128 PORT_IRQ_IF_ERR = (1 << 27), /* interface fatal error */
129 PORT_IRQ_IF_NONFATAL = (1 << 26), /* interface non-fatal error */
130 PORT_IRQ_OVERFLOW = (1 << 24), /* xfer exhausted available S/G */
131 PORT_IRQ_BAD_PMP = (1 << 23), /* incorrect port multiplier */
132
133 PORT_IRQ_PHYRDY = (1 << 22), /* PhyRdy changed */
134 PORT_IRQ_DEV_ILCK = (1 << 7), /* device interlock */
135 PORT_IRQ_CONNECT = (1 << 6), /* port connect change status */
136 PORT_IRQ_SG_DONE = (1 << 5), /* descriptor processed */
137 PORT_IRQ_UNK_FIS = (1 << 4), /* unknown FIS rx'd */
138 PORT_IRQ_SDB_FIS = (1 << 3), /* Set Device Bits FIS rx'd */
139 PORT_IRQ_DMAS_FIS = (1 << 2), /* DMA Setup FIS rx'd */
140 PORT_IRQ_PIOS_FIS = (1 << 1), /* PIO Setup FIS rx'd */
141 PORT_IRQ_D2H_REG_FIS = (1 << 0), /* D2H Register FIS rx'd */
142
78cd52d0
TH		 143 PORT_IRQ_FREEZE = PORT_IRQ_HBUS_ERR |
144 PORT_IRQ_IF_ERR |
145 PORT_IRQ_CONNECT |
4296971d 146 PORT_IRQ_PHYRDY |
78cd52d0
TH		 147 PORT_IRQ_UNK_FIS,
148 PORT_IRQ_ERROR = PORT_IRQ_FREEZE |
149 PORT_IRQ_TF_ERR |
150 PORT_IRQ_HBUS_DATA_ERR,
151 DEF_PORT_IRQ = PORT_IRQ_ERROR | PORT_IRQ_SG_DONE |
152 PORT_IRQ_SDB_FIS | PORT_IRQ_DMAS_FIS |
153 PORT_IRQ_PIOS_FIS | PORT_IRQ_D2H_REG_FIS,
1da177e4
LT		 154
155 /* PORT_CMD bits */
02eaa666 156 PORT_CMD_ATAPI = (1 << 24), /* Device is ATAPI */
1da177e4
LT		 157 PORT_CMD_LIST_ON = (1 << 15), /* cmd list DMA engine running */
158 PORT_CMD_FIS_ON = (1 << 14), /* FIS DMA engine running */
159 PORT_CMD_FIS_RX = (1 << 4), /* Enable FIS receive DMA engine */
22b49985 160 PORT_CMD_CLO = (1 << 3), /* Command list override */
1da177e4
LT		 161 PORT_CMD_POWER_ON = (1 << 2), /* Power up device */
162 PORT_CMD_SPIN_UP = (1 << 1), /* Spin up device */
163 PORT_CMD_START = (1 << 0), /* Enable port DMA engine */
164
0be0aa98 165 PORT_CMD_ICC_MASK = (0xf << 28), /* i/f ICC state mask */
1da177e4
LT		 166 PORT_CMD_ICC_ACTIVE = (0x1 << 28), /* Put i/f in active state */
167 PORT_CMD_ICC_PARTIAL = (0x2 << 28), /* Put i/f in partial state */
168 PORT_CMD_ICC_SLUMBER = (0x6 << 28), /* Put i/f in slumber state */
4b0060f4 169
bf2af2a2 170 /* ap->flags bits */
4aeb0e32
TH		 171 AHCI_FLAG_NO_NCQ = (1 << 24),
172 AHCI_FLAG_IGN_IRQ_IF_ERR = (1 << 25), /* ignore IRQ_IF_ERR */
648a88be 173 AHCI_FLAG_HONOR_PI = (1 << 26), /* honor PORTS_IMPL */
55a61604 174 AHCI_FLAG_IGN_SERR_INTERNAL = (1 << 27), /* ignore SERR_INTERNAL */
c7a42156 175 AHCI_FLAG_32BIT_ONLY = (1 << 28), /* force 32bit */
cd70c266
JG		 176 AHCI_FLAG_MV_PATA = (1 << 29), /* PATA port */
177 AHCI_FLAG_NO_MSI = (1 << 30), /* no PCI MSI */
a7384925 178 AHCI_FLAG_NO_HOTPLUG = (1 << 31), /* ignore PxSERR.DIAG.N */
1188c0d8
TH		 179
180 AHCI_FLAG_COMMON = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
181 ATA_FLAG_MMIO | ATA_FLAG_PIO_DMA |
3cadbcc0 182 ATA_FLAG_ACPI_SATA,
0c88758b 183 AHCI_LFLAG_COMMON = ATA_LFLAG_SKIP_D2H_BSY,
1da177e4
LT		 184};
185
186struct ahci_cmd_hdr {
187 u32 opts;
188 u32 status;
189 u32 tbl_addr;
190 u32 tbl_addr_hi;
191 u32 reserved[4];
192};
193
194struct ahci_sg {
195 u32 addr;
196 u32 addr_hi;
197 u32 reserved;
198 u32 flags_size;
199};
200
201struct ahci_host_priv {
d447df14
TH		 202 u32 cap; /* cap to use */
203 u32 port_map; /* port map to use */
204 u32 saved_cap; /* saved initial cap */
205 u32 saved_port_map; /* saved initial port_map */
1da177e4
LT		 206};
207
208struct ahci_port_priv {
209 struct ahci_cmd_hdr *cmd_slot;
210 dma_addr_t cmd_slot_dma;
211 void *cmd_tbl;
212 dma_addr_t cmd_tbl_dma;
1da177e4
LT		 213 void *rx_fis;
214 dma_addr_t rx_fis_dma;
0291f95f 215 /* for NCQ spurious interrupt analysis */
0291f95f
TH		 216 unsigned int ncq_saw_d2h:1;
217 unsigned int ncq_saw_dmas:1;
afb2d552 218 unsigned int ncq_saw_sdb:1;
a7384925 219 u32 intr_mask; /* interrupts to enable */
1da177e4
LT		 220};
221
da3dbb17
TH		 222static int ahci_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val);
223static int ahci_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val);
1da177e4 224static int ahci_init_one (struct pci_dev *pdev, const struct pci_device_id *ent);
9a3d9eb0 225static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc);
1da177e4 226static void ahci_irq_clear(struct ata_port *ap);
1da177e4
LT		 227static int ahci_port_start(struct ata_port *ap);
228static void ahci_port_stop(struct ata_port *ap);
1da177e4
LT		 229static void ahci_tf_read(struct ata_port *ap, struct ata_taskfile *tf);
230static void ahci_qc_prep(struct ata_queued_cmd *qc);
231static u8 ahci_check_status(struct ata_port *ap);
78cd52d0
TH		 232static void ahci_freeze(struct ata_port *ap);
233static void ahci_thaw(struct ata_port *ap);
234static void ahci_error_handler(struct ata_port *ap);
ad616ffb 235static void ahci_vt8251_error_handler(struct ata_port *ap);
78cd52d0 236static void ahci_post_internal_cmd(struct ata_queued_cmd *qc);
df69c9c5 237static int ahci_port_resume(struct ata_port *ap);
dab632e8
JG		 238static unsigned int ahci_fill_sg(struct ata_queued_cmd *qc, void *cmd_tbl);
239static void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag,
240 u32 opts);
438ac6d5 241#ifdef CONFIG_PM
c1332875 242static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg);
c1332875
TH		 243static int ahci_pci_device_suspend(struct pci_dev *pdev, pm_message_t mesg);
244static int ahci_pci_device_resume(struct pci_dev *pdev);
438ac6d5 245#endif
1da177e4 246
193515d5 247static struct scsi_host_template ahci_sht = {
1da177e4
LT		 248 .module = THIS_MODULE,
249 .name = DRV_NAME,
250 .ioctl = ata_scsi_ioctl,
251 .queuecommand = ata_scsi_queuecmd,
12fad3f9
TH		 252 .change_queue_depth = ata_scsi_change_queue_depth,
253 .can_queue = AHCI_MAX_CMDS - 1,
1da177e4
LT		 254 .this_id = ATA_SHT_THIS_ID,
255 .sg_tablesize = AHCI_MAX_SG,
1da177e4
LT		 256 .cmd_per_lun = ATA_SHT_CMD_PER_LUN,
257 .emulated = ATA_SHT_EMULATED,
258 .use_clustering = AHCI_USE_CLUSTERING,
259 .proc_name = DRV_NAME,
260 .dma_boundary = AHCI_DMA_BOUNDARY,
261 .slave_configure = ata_scsi_slave_config,
ccf68c34 262 .slave_destroy = ata_scsi_slave_destroy,
1da177e4 263 .bios_param = ata_std_bios_param,
1da177e4
LT		 264};
265
057ace5e 266static const struct ata_port_operations ahci_ops = {
1da177e4
LT		 267 .check_status = ahci_check_status,
268 .check_altstatus = ahci_check_status,
1da177e4
LT		 269 .dev_select = ata_noop_dev_select,
270
271 .tf_read = ahci_tf_read,
272
1da177e4
LT		 273 .qc_prep = ahci_qc_prep,
274 .qc_issue = ahci_qc_issue,
275
1da177e4
LT		 276 .irq_clear = ahci_irq_clear,
277
278 .scr_read = ahci_scr_read,
279 .scr_write = ahci_scr_write,
280
78cd52d0
TH		 281 .freeze = ahci_freeze,
282 .thaw = ahci_thaw,
283
284 .error_handler = ahci_error_handler,
285 .post_internal_cmd = ahci_post_internal_cmd,
286
438ac6d5 287#ifdef CONFIG_PM
c1332875
TH		 288 .port_suspend = ahci_port_suspend,
289 .port_resume = ahci_port_resume,
438ac6d5 290#endif
c1332875 291
1da177e4
LT		 292 .port_start = ahci_port_start,
293 .port_stop = ahci_port_stop,
1da177e4
LT		 294};
295
ad616ffb 296static const struct ata_port_operations ahci_vt8251_ops = {
ad616ffb
TH		 297 .check_status = ahci_check_status,
298 .check_altstatus = ahci_check_status,
299 .dev_select = ata_noop_dev_select,
300
301 .tf_read = ahci_tf_read,
302
303 .qc_prep = ahci_qc_prep,
304 .qc_issue = ahci_qc_issue,
305
ad616ffb
TH		 306 .irq_clear = ahci_irq_clear,
307
308 .scr_read = ahci_scr_read,
309 .scr_write = ahci_scr_write,
310
311 .freeze = ahci_freeze,
312 .thaw = ahci_thaw,
313
314 .error_handler = ahci_vt8251_error_handler,
315 .post_internal_cmd = ahci_post_internal_cmd,
316
438ac6d5 317#ifdef CONFIG_PM
ad616ffb
TH		 318 .port_suspend = ahci_port_suspend,
319 .port_resume = ahci_port_resume,
438ac6d5 320#endif
ad616ffb
TH		 321
322 .port_start = ahci_port_start,
323 .port_stop = ahci_port_stop,
324};
325
98ac62de 326static const struct ata_port_info ahci_port_info[] = {
1da177e4
LT		 327 /* board_ahci */
328 {
1188c0d8 329 .flags = AHCI_FLAG_COMMON,
0c88758b 330 .link_flags = AHCI_LFLAG_COMMON,
7da79312 331 .pio_mask = 0x1f, /* pio0-4 */
469248ab 332 .udma_mask = ATA_UDMA6,
1da177e4
LT		 333 .port_ops = &ahci_ops,
334 },
648a88be
TH		 335 /* board_ahci_pi */
336 {
1188c0d8 337 .flags = AHCI_FLAG_COMMON | AHCI_FLAG_HONOR_PI,
0c88758b 338 .link_flags = AHCI_LFLAG_COMMON,
648a88be 339 .pio_mask = 0x1f, /* pio0-4 */
469248ab 340 .udma_mask = ATA_UDMA6,
648a88be
TH		 341 .port_ops = &ahci_ops,
342 },
bf2af2a2
BJ		 343 /* board_ahci_vt8251 */
344 {
0c88758b
TH		 345 .flags = AHCI_FLAG_COMMON | AHCI_FLAG_NO_NCQ,
346 .link_flags = AHCI_LFLAG_COMMON | ATA_LFLAG_HRST_TO_RESUME,
bf2af2a2 347 .pio_mask = 0x1f, /* pio0-4 */
469248ab 348 .udma_mask = ATA_UDMA6,
ad616ffb 349 .port_ops = &ahci_vt8251_ops,
bf2af2a2 350 },
41669553
TH		 351 /* board_ahci_ign_iferr */
352 {
1188c0d8 353 .flags = AHCI_FLAG_COMMON | AHCI_FLAG_IGN_IRQ_IF_ERR,
0c88758b 354 .link_flags = AHCI_LFLAG_COMMON,
41669553 355 .pio_mask = 0x1f, /* pio0-4 */
469248ab 356 .udma_mask = ATA_UDMA6,
41669553
TH		 357 .port_ops = &ahci_ops,
358 },
55a61604
CH		 359 /* board_ahci_sb600 */
360 {
1188c0d8 361 .flags = AHCI_FLAG_COMMON |
c7a42156
TH		 362 AHCI_FLAG_IGN_SERR_INTERNAL |
363 AHCI_FLAG_32BIT_ONLY,
0c88758b 364 .link_flags = AHCI_LFLAG_COMMON,
55a61604 365 .pio_mask = 0x1f, /* pio0-4 */
469248ab 366 .udma_mask = ATA_UDMA6,
55a61604
CH		 367 .port_ops = &ahci_ops,
368 },
cd70c266
JG		 369 /* board_ahci_mv */
370 {
371 .sht = &ahci_sht,
372 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
373 ATA_FLAG_MMIO | ATA_FLAG_PIO_DMA |
0c88758b
TH		 374 AHCI_FLAG_HONOR_PI | AHCI_FLAG_NO_NCQ |
375 AHCI_FLAG_NO_MSI | AHCI_FLAG_MV_PATA,
376 .link_flags = AHCI_LFLAG_COMMON,
cd70c266
JG		 377 .pio_mask = 0x1f, /* pio0-4 */
378 .udma_mask = ATA_UDMA6,
379 .port_ops = &ahci_ops,
380 },
1da177e4
LT		 381};
382
3b7d697d 383static const struct pci_device_id ahci_pci_tbl[] = {
fe7fa31a 384 /* Intel */
54bb3a94
JG		 385 { PCI_VDEVICE(INTEL, 0x2652), board_ahci }, /* ICH6 */
386 { PCI_VDEVICE(INTEL, 0x2653), board_ahci }, /* ICH6M */
387 { PCI_VDEVICE(INTEL, 0x27c1), board_ahci }, /* ICH7 */
388 { PCI_VDEVICE(INTEL, 0x27c5), board_ahci }, /* ICH7M */
389 { PCI_VDEVICE(INTEL, 0x27c3), board_ahci }, /* ICH7R */
82490c09 390 { PCI_VDEVICE(AL, 0x5288), board_ahci_ign_iferr }, /* ULi M5288 */
54bb3a94
JG		 391 { PCI_VDEVICE(INTEL, 0x2681), board_ahci }, /* ESB2 */
392 { PCI_VDEVICE(INTEL, 0x2682), board_ahci }, /* ESB2 */
393 { PCI_VDEVICE(INTEL, 0x2683), board_ahci }, /* ESB2 */
394 { PCI_VDEVICE(INTEL, 0x27c6), board_ahci }, /* ICH7-M DH */
648a88be
TH		 395 { PCI_VDEVICE(INTEL, 0x2821), board_ahci_pi }, /* ICH8 */
396 { PCI_VDEVICE(INTEL, 0x2822), board_ahci_pi }, /* ICH8 */
397 { PCI_VDEVICE(INTEL, 0x2824), board_ahci_pi }, /* ICH8 */
398 { PCI_VDEVICE(INTEL, 0x2829), board_ahci_pi }, /* ICH8M */
399 { PCI_VDEVICE(INTEL, 0x282a), board_ahci_pi }, /* ICH8M */
400 { PCI_VDEVICE(INTEL, 0x2922), board_ahci_pi }, /* ICH9 */
401 { PCI_VDEVICE(INTEL, 0x2923), board_ahci_pi }, /* ICH9 */
402 { PCI_VDEVICE(INTEL, 0x2924), board_ahci_pi }, /* ICH9 */
403 { PCI_VDEVICE(INTEL, 0x2925), board_ahci_pi }, /* ICH9 */
404 { PCI_VDEVICE(INTEL, 0x2927), board_ahci_pi }, /* ICH9 */
405 { PCI_VDEVICE(INTEL, 0x2929), board_ahci_pi }, /* ICH9M */
406 { PCI_VDEVICE(INTEL, 0x292a), board_ahci_pi }, /* ICH9M */
407 { PCI_VDEVICE(INTEL, 0x292b), board_ahci_pi }, /* ICH9M */
8af12cdb 408 { PCI_VDEVICE(INTEL, 0x292c), board_ahci_pi }, /* ICH9M */
648a88be
TH		 409 { PCI_VDEVICE(INTEL, 0x292f), board_ahci_pi }, /* ICH9M */
410 { PCI_VDEVICE(INTEL, 0x294d), board_ahci_pi }, /* ICH9 */
411 { PCI_VDEVICE(INTEL, 0x294e), board_ahci_pi }, /* ICH9M */
fe7fa31a 412
e34bb370
TH		 413 /* JMicron 360/1/3/5/6, match class to avoid IDE function */
414 { PCI_VENDOR_ID_JMICRON, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
415 PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci_ign_iferr },
fe7fa31a
JG		 416
417 /* ATI */
c65ec1c2 418 { PCI_VDEVICE(ATI, 0x4380), board_ahci_sb600 }, /* ATI SB600 */
c69c0892 419 { PCI_VDEVICE(ATI, 0x4390), board_ahci_sb600 }, /* ATI SB700/800 */
420 { PCI_VDEVICE(ATI, 0x4391), board_ahci_sb600 }, /* ATI SB700/800 */
421 { PCI_VDEVICE(ATI, 0x4392), board_ahci_sb600 }, /* ATI SB700/800 */
422 { PCI_VDEVICE(ATI, 0x4393), board_ahci_sb600 }, /* ATI SB700/800 */
423 { PCI_VDEVICE(ATI, 0x4394), board_ahci_sb600 }, /* ATI SB700/800 */
424 { PCI_VDEVICE(ATI, 0x4395), board_ahci_sb600 }, /* ATI SB700/800 */
fe7fa31a
JG		 425
426 /* VIA */
54bb3a94 427 { PCI_VDEVICE(VIA, 0x3349), board_ahci_vt8251 }, /* VIA VT8251 */
bf335542 428 { PCI_VDEVICE(VIA, 0x6287), board_ahci_vt8251 }, /* VIA VT8251 */
fe7fa31a
JG		 429
430 /* NVIDIA */
54bb3a94
JG		 431 { PCI_VDEVICE(NVIDIA, 0x044c), board_ahci }, /* MCP65 */
432 { PCI_VDEVICE(NVIDIA, 0x044d), board_ahci }, /* MCP65 */
433 { PCI_VDEVICE(NVIDIA, 0x044e), board_ahci }, /* MCP65 */
434 { PCI_VDEVICE(NVIDIA, 0x044f), board_ahci }, /* MCP65 */
6fbf5ba4
PC		 435 { PCI_VDEVICE(NVIDIA, 0x045c), board_ahci }, /* MCP65 */
436 { PCI_VDEVICE(NVIDIA, 0x045d), board_ahci }, /* MCP65 */
437 { PCI_VDEVICE(NVIDIA, 0x045e), board_ahci }, /* MCP65 */
438 { PCI_VDEVICE(NVIDIA, 0x045f), board_ahci }, /* MCP65 */
439 { PCI_VDEVICE(NVIDIA, 0x0550), board_ahci }, /* MCP67 */
440 { PCI_VDEVICE(NVIDIA, 0x0551), board_ahci }, /* MCP67 */
441 { PCI_VDEVICE(NVIDIA, 0x0552), board_ahci }, /* MCP67 */
442 { PCI_VDEVICE(NVIDIA, 0x0553), board_ahci }, /* MCP67 */
895663cd
PC		 443 { PCI_VDEVICE(NVIDIA, 0x0554), board_ahci }, /* MCP67 */
444 { PCI_VDEVICE(NVIDIA, 0x0555), board_ahci }, /* MCP67 */
445 { PCI_VDEVICE(NVIDIA, 0x0556), board_ahci }, /* MCP67 */
446 { PCI_VDEVICE(NVIDIA, 0x0557), board_ahci }, /* MCP67 */
447 { PCI_VDEVICE(NVIDIA, 0x0558), board_ahci }, /* MCP67 */
448 { PCI_VDEVICE(NVIDIA, 0x0559), board_ahci }, /* MCP67 */
449 { PCI_VDEVICE(NVIDIA, 0x055a), board_ahci }, /* MCP67 */
450 { PCI_VDEVICE(NVIDIA, 0x055b), board_ahci }, /* MCP67 */
0522b286
PC		 451 { PCI_VDEVICE(NVIDIA, 0x07f0), board_ahci }, /* MCP73 */
452 { PCI_VDEVICE(NVIDIA, 0x07f1), board_ahci }, /* MCP73 */
453 { PCI_VDEVICE(NVIDIA, 0x07f2), board_ahci }, /* MCP73 */
454 { PCI_VDEVICE(NVIDIA, 0x07f3), board_ahci }, /* MCP73 */
455 { PCI_VDEVICE(NVIDIA, 0x07f4), board_ahci }, /* MCP73 */
456 { PCI_VDEVICE(NVIDIA, 0x07f5), board_ahci }, /* MCP73 */
457 { PCI_VDEVICE(NVIDIA, 0x07f6), board_ahci }, /* MCP73 */
458 { PCI_VDEVICE(NVIDIA, 0x07f7), board_ahci }, /* MCP73 */
459 { PCI_VDEVICE(NVIDIA, 0x07f8), board_ahci }, /* MCP73 */
460 { PCI_VDEVICE(NVIDIA, 0x07f9), board_ahci }, /* MCP73 */
461 { PCI_VDEVICE(NVIDIA, 0x07fa), board_ahci }, /* MCP73 */
462 { PCI_VDEVICE(NVIDIA, 0x07fb), board_ahci }, /* MCP73 */
463 { PCI_VDEVICE(NVIDIA, 0x0ad0), board_ahci }, /* MCP77 */
464 { PCI_VDEVICE(NVIDIA, 0x0ad1), board_ahci }, /* MCP77 */
465 { PCI_VDEVICE(NVIDIA, 0x0ad2), board_ahci }, /* MCP77 */
466 { PCI_VDEVICE(NVIDIA, 0x0ad3), board_ahci }, /* MCP77 */
467 { PCI_VDEVICE(NVIDIA, 0x0ad4), board_ahci }, /* MCP77 */
468 { PCI_VDEVICE(NVIDIA, 0x0ad5), board_ahci }, /* MCP77 */
469 { PCI_VDEVICE(NVIDIA, 0x0ad6), board_ahci }, /* MCP77 */
470 { PCI_VDEVICE(NVIDIA, 0x0ad7), board_ahci }, /* MCP77 */
471 { PCI_VDEVICE(NVIDIA, 0x0ad8), board_ahci }, /* MCP77 */
472 { PCI_VDEVICE(NVIDIA, 0x0ad9), board_ahci }, /* MCP77 */
473 { PCI_VDEVICE(NVIDIA, 0x0ada), board_ahci }, /* MCP77 */
474 { PCI_VDEVICE(NVIDIA, 0x0adb), board_ahci }, /* MCP77 */
fe7fa31a 475
95916edd 476 /* SiS */
54bb3a94
JG		 477 { PCI_VDEVICE(SI, 0x1184), board_ahci }, /* SiS 966 */
478 { PCI_VDEVICE(SI, 0x1185), board_ahci }, /* SiS 966 */
479 { PCI_VDEVICE(SI, 0x0186), board_ahci }, /* SiS 968 */
95916edd 480
cd70c266
JG		 481 /* Marvell */
482 { PCI_VDEVICE(MARVELL, 0x6145), board_ahci_mv }, /* 6145 */
483
415ae2b5
JG		 484 /* Generic, PCI class code for AHCI */
485 { PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
c9f89475 486 PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci },
415ae2b5 487
1da177e4
LT		 488 { } /* terminate list */
489};
490
491
492static struct pci_driver ahci_pci_driver = {
493 .name = DRV_NAME,
494 .id_table = ahci_pci_tbl,
495 .probe = ahci_init_one,
24dc5f33 496 .remove = ata_pci_remove_one,
438ac6d5 497#ifdef CONFIG_PM
c1332875
TH		 498 .suspend = ahci_pci_device_suspend,
499 .resume = ahci_pci_device_resume,
438ac6d5 500#endif
1da177e4
LT		 501};
502
503
98fa4b60
TH		 504static inline int ahci_nr_ports(u32 cap)
505{
506 return (cap & 0x1f) + 1;
507}
508
dab632e8
JG		 509static inline void __iomem *__ahci_port_base(struct ata_host *host,
510 unsigned int port_no)
1da177e4 511{
dab632e8 512 void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
4447d351 513
dab632e8
JG		 514 return mmio + 0x100 + (port_no * 0x80);
515}
516
517static inline void __iomem *ahci_port_base(struct ata_port *ap)
518{
519 return __ahci_port_base(ap->host, ap->port_no);
1da177e4
LT		 520}
521
d447df14
TH		 522/**
523 * ahci_save_initial_config - Save and fixup initial config values
4447d351
TH		 524 * @pdev: target PCI device
525 * @pi: associated ATA port info
526 * @hpriv: host private area to store config values
d447df14
TH		 527 *
528 * Some registers containing configuration info might be setup by
529 * BIOS and might be cleared on reset. This function saves the
530 * initial values of those registers into @hpriv such that they
531 * can be restored after controller reset.
532 *
533 * If inconsistent, config values are fixed up by this function.
534 *
535 * LOCKING:
536 * None.
537 */
4447d351
TH		 538static void ahci_save_initial_config(struct pci_dev *pdev,
539 const struct ata_port_info *pi,
540 struct ahci_host_priv *hpriv)
d447df14 541{
4447d351 542 void __iomem *mmio = pcim_iomap_table(pdev)[AHCI_PCI_BAR];
d447df14 543 u32 cap, port_map;
17199b18 544 int i;
d447df14
TH		 545
546 /* Values prefixed with saved_ are written back to host after
547 * reset. Values without are used for driver operation.
548 */
549 hpriv->saved_cap = cap = readl(mmio + HOST_CAP);
550 hpriv->saved_port_map = port_map = readl(mmio + HOST_PORTS_IMPL);
551
274c1fde 552 /* some chips have errata preventing 64bit use */
c7a42156
TH		 553 if ((cap & HOST_CAP_64) && (pi->flags & AHCI_FLAG_32BIT_ONLY)) {
554 dev_printk(KERN_INFO, &pdev->dev,
555 "controller can't do 64bit DMA, forcing 32bit\n");
556 cap &= ~HOST_CAP_64;
557 }
558
274c1fde
TH		 559 if ((cap & HOST_CAP_NCQ) && (pi->flags & AHCI_FLAG_NO_NCQ)) {
560 dev_printk(KERN_INFO, &pdev->dev,
561 "controller can't do NCQ, turning off CAP_NCQ\n");
562 cap &= ~HOST_CAP_NCQ;
563 }
564
d447df14
TH		 565 /* fixup zero port_map */
566 if (!port_map) {
a3d2cc5e 567 port_map = (1 << ahci_nr_ports(cap)) - 1;
4447d351 568 dev_printk(KERN_WARNING, &pdev->dev,
d447df14
TH		 569 "PORTS_IMPL is zero, forcing 0x%x\n", port_map);
570
571 /* write the fixed up value to the PI register */
572 hpriv->saved_port_map = port_map;
573 }
574
cd70c266
JG		 575 /*
576 * Temporary Marvell 6145 hack: PATA port presence
577 * is asserted through the standard AHCI port
578 * presence register, as bit 4 (counting from 0)
579 */
580 if (pi->flags & AHCI_FLAG_MV_PATA) {
581 dev_printk(KERN_ERR, &pdev->dev,
582 "MV_AHCI HACK: port_map %x -> %x\n",
583 hpriv->port_map,
584 hpriv->port_map & 0xf);
585
586 port_map &= 0xf;
587 }
588
17199b18 589 /* cross check port_map and cap.n_ports */
4447d351 590 if (pi->flags & AHCI_FLAG_HONOR_PI) {
17199b18
TH		 591 u32 tmp_port_map = port_map;
592 int n_ports = ahci_nr_ports(cap);
593
594 for (i = 0; i < AHCI_MAX_PORTS && n_ports; i++) {
595 if (tmp_port_map & (1 << i)) {
596 n_ports--;
597 tmp_port_map &= ~(1 << i);
598 }
599 }
600
601 /* Whine if inconsistent. No need to update cap.
602 * port_map is used to determine number of ports.
603 */
604 if (n_ports || tmp_port_map)
4447d351 605 dev_printk(KERN_WARNING, &pdev->dev,
17199b18
TH		 606 "nr_ports (%u) and implemented port map "
607 "(0x%x) don't match\n",
608 ahci_nr_ports(cap), port_map);
609 } else {
610 /* fabricate port_map from cap.nr_ports */
611 port_map = (1 << ahci_nr_ports(cap)) - 1;
612 }
613
d447df14
TH		 614 /* record values to use during operation */
615 hpriv->cap = cap;
616 hpriv->port_map = port_map;
617}
618
619/**
620 * ahci_restore_initial_config - Restore initial config
4447d351 621 * @host: target ATA host
d447df14
TH		 622 *
623 * Restore initial config stored by ahci_save_initial_config().
624 *
625 * LOCKING:
626 * None.
627 */
4447d351 628static void ahci_restore_initial_config(struct ata_host *host)
d447df14 629{
4447d351
TH		 630 struct ahci_host_priv *hpriv = host->private_data;
631 void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
632
d447df14
TH		 633 writel(hpriv->saved_cap, mmio + HOST_CAP);
634 writel(hpriv->saved_port_map, mmio + HOST_PORTS_IMPL);
635 (void) readl(mmio + HOST_PORTS_IMPL); /* flush */
636}
637
203ef6c4 638static unsigned ahci_scr_offset(struct ata_port *ap, unsigned int sc_reg)
1da177e4 639{
203ef6c4
TH		 640 static const int offset[] = {
641 [SCR_STATUS] = PORT_SCR_STAT,
642 [SCR_CONTROL] = PORT_SCR_CTL,
643 [SCR_ERROR] = PORT_SCR_ERR,
644 [SCR_ACTIVE] = PORT_SCR_ACT,
645 [SCR_NOTIFICATION] = PORT_SCR_NTF,
646 };
647 struct ahci_host_priv *hpriv = ap->host->private_data;
1da177e4 648
203ef6c4
TH		 649 if (sc_reg < ARRAY_SIZE(offset) &&
650 (sc_reg != SCR_NOTIFICATION || (hpriv->cap & HOST_CAP_SNTF)))
651 return offset[sc_reg];
da3dbb17 652 return 0;
1da177e4
LT		 653}
654
203ef6c4 655static int ahci_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val)
1da177e4 656{
203ef6c4
TH		 657 void __iomem *port_mmio = ahci_port_base(ap);
658 int offset = ahci_scr_offset(ap, sc_reg);
659
660 if (offset) {
661 *val = readl(port_mmio + offset);
662 return 0;
1da177e4 663 }
203ef6c4
TH		 664 return -EINVAL;
665}
1da177e4 666
203ef6c4
TH		 667static int ahci_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val)
668{
669 void __iomem *port_mmio = ahci_port_base(ap);
670 int offset = ahci_scr_offset(ap, sc_reg);
671
672 if (offset) {
673 writel(val, port_mmio + offset);
674 return 0;
675 }
676 return -EINVAL;
1da177e4
LT		 677}
678
4447d351 679static void ahci_start_engine(struct ata_port *ap)
7c76d1e8 680{
4447d351 681 void __iomem *port_mmio = ahci_port_base(ap);
7c76d1e8
TH		 682 u32 tmp;
683
d8fcd116 684 /* start DMA */
9f592056 685 tmp = readl(port_mmio + PORT_CMD);
7c76d1e8
TH		 686 tmp |= PORT_CMD_START;
687 writel(tmp, port_mmio + PORT_CMD);
688 readl(port_mmio + PORT_CMD); /* flush */
689}
690
4447d351 691static int ahci_stop_engine(struct ata_port *ap)
254950cd 692{
4447d351 693 void __iomem *port_mmio = ahci_port_base(ap);
254950cd
TH		 694 u32 tmp;
695
696 tmp = readl(port_mmio + PORT_CMD);
697
d8fcd116 698 /* check if the HBA is idle */
254950cd
TH		 699 if ((tmp & (PORT_CMD_START | PORT_CMD_LIST_ON)) == 0)
700 return 0;
701
d8fcd116 702 /* setting HBA to idle */
254950cd
TH		 703 tmp &= ~PORT_CMD_START;
704 writel(tmp, port_mmio + PORT_CMD);
705
d8fcd116 706 /* wait for engine to stop. This could be as long as 500 msec */
254950cd
TH		 707 tmp = ata_wait_register(port_mmio + PORT_CMD,
708 PORT_CMD_LIST_ON, PORT_CMD_LIST_ON, 1, 500);
d8fcd116 709 if (tmp & PORT_CMD_LIST_ON)
254950cd
TH		 710 return -EIO;
711
712 return 0;
713}
714
4447d351 715static void ahci_start_fis_rx(struct ata_port *ap)
0be0aa98 716{
4447d351
TH		 717 void __iomem *port_mmio = ahci_port_base(ap);
718 struct ahci_host_priv *hpriv = ap->host->private_data;
719 struct ahci_port_priv *pp = ap->private_data;
0be0aa98
TH		 720 u32 tmp;
721
722 /* set FIS registers */
4447d351
TH		 723 if (hpriv->cap & HOST_CAP_64)
724 writel((pp->cmd_slot_dma >> 16) >> 16,
725 port_mmio + PORT_LST_ADDR_HI);
726 writel(pp->cmd_slot_dma & 0xffffffff, port_mmio + PORT_LST_ADDR);
0be0aa98 727
4447d351
TH		 728 if (hpriv->cap & HOST_CAP_64)
729 writel((pp->rx_fis_dma >> 16) >> 16,
730 port_mmio + PORT_FIS_ADDR_HI);
731 writel(pp->rx_fis_dma & 0xffffffff, port_mmio + PORT_FIS_ADDR);
0be0aa98
TH		 732
733 /* enable FIS reception */
734 tmp = readl(port_mmio + PORT_CMD);
735 tmp |= PORT_CMD_FIS_RX;
736 writel(tmp, port_mmio + PORT_CMD);
737
738 /* flush */
739 readl(port_mmio + PORT_CMD);
740}
741
4447d351 742static int ahci_stop_fis_rx(struct ata_port *ap)
0be0aa98 743{
4447d351 744 void __iomem *port_mmio = ahci_port_base(ap);
0be0aa98
TH		 745 u32 tmp;
746
747 /* disable FIS reception */
748 tmp = readl(port_mmio + PORT_CMD);
749 tmp &= ~PORT_CMD_FIS_RX;
750 writel(tmp, port_mmio + PORT_CMD);
751
752 /* wait for completion, spec says 500ms, give it 1000 */
753 tmp = ata_wait_register(port_mmio + PORT_CMD, PORT_CMD_FIS_ON,
754 PORT_CMD_FIS_ON, 10, 1000);
755 if (tmp & PORT_CMD_FIS_ON)
756 return -EBUSY;
757
758 return 0;
759}
760
4447d351 761static void ahci_power_up(struct ata_port *ap)
0be0aa98 762{
4447d351
TH		 763 struct ahci_host_priv *hpriv = ap->host->private_data;
764 void __iomem *port_mmio = ahci_port_base(ap);
0be0aa98
TH		 765 u32 cmd;
766
767 cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
768
769 /* spin up device */
4447d351 770 if (hpriv->cap & HOST_CAP_SSS) {
0be0aa98
TH		 771 cmd |= PORT_CMD_SPIN_UP;
772 writel(cmd, port_mmio + PORT_CMD);
773 }
774
775 /* wake up link */
776 writel(cmd | PORT_CMD_ICC_ACTIVE, port_mmio + PORT_CMD);
777}
778
438ac6d5 779#ifdef CONFIG_PM
4447d351 780static void ahci_power_down(struct ata_port *ap)
0be0aa98 781{
4447d351
TH		 782 struct ahci_host_priv *hpriv = ap->host->private_data;
783 void __iomem *port_mmio = ahci_port_base(ap);
0be0aa98
TH		 784 u32 cmd, scontrol;
785
4447d351 786 if (!(hpriv->cap & HOST_CAP_SSS))
07c53dac 787 return;
0be0aa98 788
07c53dac
TH		 789 /* put device into listen mode, first set PxSCTL.DET to 0 */
790 scontrol = readl(port_mmio + PORT_SCR_CTL);
791 scontrol &= ~0xf;
792 writel(scontrol, port_mmio + PORT_SCR_CTL);
0be0aa98 793
07c53dac
TH		 794 /* then set PxCMD.SUD to 0 */
795 cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
796 cmd &= ~PORT_CMD_SPIN_UP;
797 writel(cmd, port_mmio + PORT_CMD);
0be0aa98 798}
438ac6d5 799#endif
0be0aa98 800
df69c9c5 801static void ahci_start_port(struct ata_port *ap)
0be0aa98 802{
0be0aa98 803 /* enable FIS reception */
4447d351 804 ahci_start_fis_rx(ap);
0be0aa98
TH		 805
806 /* enable DMA */
4447d351 807 ahci_start_engine(ap);
0be0aa98
TH		 808}
809
4447d351 810static int ahci_deinit_port(struct ata_port *ap, const char **emsg)
0be0aa98
TH		 811{
812 int rc;
813
814 /* disable DMA */
4447d351 815 rc = ahci_stop_engine(ap);
0be0aa98
TH		 816 if (rc) {
817 *emsg = "failed to stop engine";
818 return rc;
819 }
820
821 /* disable FIS reception */
4447d351 822 rc = ahci_stop_fis_rx(ap);
0be0aa98
TH		 823 if (rc) {
824 *emsg = "failed stop FIS RX";
825 return rc;
826 }
827
0be0aa98
TH		 828 return 0;
829}
830
4447d351 831static int ahci_reset_controller(struct ata_host *host)
d91542c1 832{
4447d351
TH		 833 struct pci_dev *pdev = to_pci_dev(host->dev);
834 void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
d447df14 835 u32 tmp;
d91542c1
TH		 836
837 /* global controller reset */
838 tmp = readl(mmio + HOST_CTL);
839 if ((tmp & HOST_RESET) == 0) {
840 writel(tmp | HOST_RESET, mmio + HOST_CTL);
841 readl(mmio + HOST_CTL); /* flush */
842 }
843
844 /* reset must complete within 1 second, or
845 * the hardware should be considered fried.
846 */
847 ssleep(1);
848
849 tmp = readl(mmio + HOST_CTL);
850 if (tmp & HOST_RESET) {
4447d351 851 dev_printk(KERN_ERR, host->dev,
d91542c1
TH		 852 "controller reset failed (0x%x)\n", tmp);
853 return -EIO;
854 }
855
98fa4b60 856 /* turn on AHCI mode */
d91542c1
TH		 857 writel(HOST_AHCI_EN, mmio + HOST_CTL);
858 (void) readl(mmio + HOST_CTL); /* flush */
98fa4b60 859
d447df14 860 /* some registers might be cleared on reset. restore initial values */
4447d351 861 ahci_restore_initial_config(host);
d91542c1
TH		 862
863 if (pdev->vendor == PCI_VENDOR_ID_INTEL) {
864 u16 tmp16;
865
866 /* configure PCS */
867 pci_read_config_word(pdev, 0x92, &tmp16);
868 tmp16 |= 0xf;
869 pci_write_config_word(pdev, 0x92, tmp16);
870 }
871
872 return 0;
873}
874
2bcd866b
JG		 875static void ahci_port_init(struct pci_dev *pdev, struct ata_port *ap,
876 int port_no, void __iomem *mmio,
877 void __iomem *port_mmio)
878{
879 const char *emsg = NULL;
880 int rc;
881 u32 tmp;
882
883 /* make sure port is not active */
884 rc = ahci_deinit_port(ap, &emsg);
885 if (rc)
886 dev_printk(KERN_WARNING, &pdev->dev,
887 "%s (%d)\n", emsg, rc);
888
889 /* clear SError */
890 tmp = readl(port_mmio + PORT_SCR_ERR);
891 VPRINTK("PORT_SCR_ERR 0x%x\n", tmp);
892 writel(tmp, port_mmio + PORT_SCR_ERR);
893
894 /* clear port IRQ */
895 tmp = readl(port_mmio + PORT_IRQ_STAT);
896 VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp);
897 if (tmp)
898 writel(tmp, port_mmio + PORT_IRQ_STAT);
899
900 writel(1 << port_no, mmio + HOST_IRQ_STAT);
901}
902
4447d351 903static void ahci_init_controller(struct ata_host *host)
d91542c1 904{
4447d351
TH		 905 struct pci_dev *pdev = to_pci_dev(host->dev);
906 void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
2bcd866b 907 int i;
cd70c266 908 void __iomem *port_mmio;
d91542c1
TH		 909 u32 tmp;
910
cd70c266
JG		 911 if (host->ports[0]->flags & AHCI_FLAG_MV_PATA) {
912 port_mmio = __ahci_port_base(host, 4);
913
914 writel(0, port_mmio + PORT_IRQ_MASK);
915
916 /* clear port IRQ */
917 tmp = readl(port_mmio + PORT_IRQ_STAT);
918 VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp);
919 if (tmp)
920 writel(tmp, port_mmio + PORT_IRQ_STAT);
921 }
922
4447d351
TH		 923 for (i = 0; i < host->n_ports; i++) {
924 struct ata_port *ap = host->ports[i];
d91542c1 925
cd70c266 926 port_mmio = ahci_port_base(ap);
4447d351 927 if (ata_port_is_dummy(ap))
d91542c1 928 continue;
d91542c1 929
2bcd866b 930 ahci_port_init(pdev, ap, i, mmio, port_mmio);
d91542c1
TH		 931 }
932
933 tmp = readl(mmio + HOST_CTL);
934 VPRINTK("HOST_CTL 0x%x\n", tmp);
935 writel(tmp | HOST_IRQ_EN, mmio + HOST_CTL);
936 tmp = readl(mmio + HOST_CTL);
937 VPRINTK("HOST_CTL 0x%x\n", tmp);
938}
939
422b7595 940static unsigned int ahci_dev_classify(struct ata_port *ap)
1da177e4 941{
4447d351 942 void __iomem *port_mmio = ahci_port_base(ap);
1da177e4 943 struct ata_taskfile tf;
422b7595
TH		 944 u32 tmp;
945
946 tmp = readl(port_mmio + PORT_SIG);
947 tf.lbah = (tmp >> 24) & 0xff;
948 tf.lbam = (tmp >> 16) & 0xff;
949 tf.lbal = (tmp >> 8) & 0xff;
950 tf.nsect = (tmp) & 0xff;
951
952 return ata_dev_classify(&tf);
953}
954
12fad3f9
TH		 955static void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag,
956 u32 opts)
cc9278ed 957{
12fad3f9
TH		 958 dma_addr_t cmd_tbl_dma;
959
960 cmd_tbl_dma = pp->cmd_tbl_dma + tag * AHCI_CMD_TBL_SZ;
961
962 pp->cmd_slot[tag].opts = cpu_to_le32(opts);
963 pp->cmd_slot[tag].status = 0;
964 pp->cmd_slot[tag].tbl_addr = cpu_to_le32(cmd_tbl_dma & 0xffffffff);
965 pp->cmd_slot[tag].tbl_addr_hi = cpu_to_le32((cmd_tbl_dma >> 16) >> 16);
cc9278ed
TH		 966}
967
d2e75dff 968static int ahci_kick_engine(struct ata_port *ap, int force_restart)
4658f79b 969{
0d5ff566 970 void __iomem *port_mmio = ap->ioaddr.cmd_addr;
cca3974e 971 struct ahci_host_priv *hpriv = ap->host->private_data;
bf2af2a2 972 u32 tmp;
d2e75dff 973 int busy, rc;
bf2af2a2 974
d2e75dff
TH		 975 /* do we need to kick the port? */
976 busy = ahci_check_status(ap) & (ATA_BUSY | ATA_DRQ);
977 if (!busy && !force_restart)
978 return 0;
979
980 /* stop engine */
981 rc = ahci_stop_engine(ap);
982 if (rc)
983 goto out_restart;
984
985 /* need to do CLO? */
986 if (!busy) {
987 rc = 0;
988 goto out_restart;
989 }
990
991 if (!(hpriv->cap & HOST_CAP_CLO)) {
992 rc = -EOPNOTSUPP;
993 goto out_restart;
994 }
bf2af2a2 995
d2e75dff 996 /* perform CLO */
bf2af2a2
BJ		 997 tmp = readl(port_mmio + PORT_CMD);
998 tmp |= PORT_CMD_CLO;
999 writel(tmp, port_mmio + PORT_CMD);
1000
d2e75dff 1001 rc = 0;
bf2af2a2
BJ		 1002 tmp = ata_wait_register(port_mmio + PORT_CMD,
1003 PORT_CMD_CLO, PORT_CMD_CLO, 1, 500);
1004 if (tmp & PORT_CMD_CLO)
d2e75dff 1005 rc = -EIO;
bf2af2a2 1006
d2e75dff
TH		 1007 /* restart engine */
1008 out_restart:
1009 ahci_start_engine(ap);
1010 return rc;
bf2af2a2
BJ		 1011}
1012
91c4a2e0
TH		 1013static int ahci_exec_polled_cmd(struct ata_port *ap, int pmp,
1014 struct ata_taskfile *tf, int is_cmd, u16 flags,
1015 unsigned long timeout_msec)
bf2af2a2 1016{
91c4a2e0 1017 const u32 cmd_fis_len = 5; /* five dwords */
4658f79b 1018 struct ahci_port_priv *pp = ap->private_data;
4447d351 1019 void __iomem *port_mmio = ahci_port_base(ap);
91c4a2e0
TH		 1020 u8 *fis = pp->cmd_tbl;
1021 u32 tmp;
1022
1023 /* prep the command */
1024 ata_tf_to_fis(tf, pmp, is_cmd, fis);
1025 ahci_fill_cmd_slot(pp, 0, cmd_fis_len | flags | (pmp << 12));
1026
1027 /* issue & wait */
1028 writel(1, port_mmio + PORT_CMD_ISSUE);
1029
1030 if (timeout_msec) {
1031 tmp = ata_wait_register(port_mmio + PORT_CMD_ISSUE, 0x1, 0x1,
1032 1, timeout_msec);
1033 if (tmp & 0x1) {
1034 ahci_kick_engine(ap, 1);
1035 return -EBUSY;
1036 }
1037 } else
1038 readl(port_mmio + PORT_CMD_ISSUE); /* flush */
1039
1040 return 0;
1041}
1042
cc0680a5 1043static int ahci_do_softreset(struct ata_link *link, unsigned int *class,
a9cf5e85 1044 int pmp, unsigned long deadline)
91c4a2e0 1045{
cc0680a5 1046 struct ata_port *ap = link->ap;
4658f79b 1047 const char *reason = NULL;
2cbb79eb 1048 unsigned long now, msecs;
4658f79b 1049 struct ata_taskfile tf;
4658f79b
TH		 1050 int rc;
1051
1052 DPRINTK("ENTER\n");
1053
cc0680a5 1054 if (ata_link_offline(link)) {
c2a65852
TH		 1055 DPRINTK("PHY reports no device\n");
1056 *class = ATA_DEV_NONE;
1057 return 0;
1058 }
1059
4658f79b 1060 /* prepare for SRST (AHCI-1.1 10.4.1) */
d2e75dff
TH		 1061 rc = ahci_kick_engine(ap, 1);
1062 if (rc)
cc0680a5 1063 ata_link_printk(link, KERN_WARNING,
d2e75dff 1064 "failed to reset engine (errno=%d)", rc);
4658f79b 1065
cc0680a5 1066 ata_tf_init(link->device, &tf);
4658f79b
TH		 1067
1068 /* issue the first D2H Register FIS */
2cbb79eb
TH		 1069 msecs = 0;
1070 now = jiffies;
1071 if (time_after(now, deadline))
1072 msecs = jiffies_to_msecs(deadline - now);
1073
4658f79b 1074 tf.ctl |= ATA_SRST;
a9cf5e85 1075 if (ahci_exec_polled_cmd(ap, pmp, &tf, 0,
91c4a2e0 1076 AHCI_CMD_RESET | AHCI_CMD_CLR_BUSY, msecs)) {
4658f79b
TH		 1077 rc = -EIO;
1078 reason = "1st FIS failed";
1079 goto fail;
1080 }
1081
1082 /* spec says at least 5us, but be generous and sleep for 1ms */
1083 msleep(1);
1084
1085 /* issue the second D2H Register FIS */
4658f79b 1086 tf.ctl &= ~ATA_SRST;
a9cf5e85 1087 ahci_exec_polled_cmd(ap, pmp, &tf, 0, 0, 0);
4658f79b
TH		 1088
1089 /* spec mandates ">= 2ms" before checking status.
1090 * We wait 150ms, because that was the magic delay used for
1091 * ATAPI devices in Hale Landis's ATADRVR, for the period of time
1092 * between when the ATA command register is written, and then
1093 * status is checked. Because waiting for "a while" before
1094 * checking status is fine, post SRST, we perform this magic
1095 * delay here as well.
1096 */
1097 msleep(150);
1098
9b89391c
TH		 1099 rc = ata_wait_ready(ap, deadline);
1100 /* link occupied, -ENODEV too is an error */
1101 if (rc) {
1102 reason = "device not ready";
1103 goto fail;
4658f79b 1104 }
9b89391c 1105 *class = ahci_dev_classify(ap);
4658f79b
TH		 1106
1107 DPRINTK("EXIT, class=%u\n", *class);
1108 return 0;
1109
4658f79b 1110 fail:
cc0680a5 1111 ata_link_printk(link, KERN_ERR, "softreset failed (%s)\n", reason);
4658f79b
TH		 1112 return rc;
1113}
1114
cc0680a5 1115static int ahci_softreset(struct ata_link *link, unsigned int *class,
a9cf5e85
TH		 1116 unsigned long deadline)
1117{
cc0680a5 1118 return ahci_do_softreset(link, class, 0, deadline);
a9cf5e85
TH		 1119}
1120
cc0680a5 1121static int ahci_hardreset(struct ata_link *link, unsigned int *class,
d4b2bab4 1122 unsigned long deadline)
422b7595 1123{
cc0680a5 1124 struct ata_port *ap = link->ap;
4296971d
TH		 1125 struct ahci_port_priv *pp = ap->private_data;
1126 u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
1127 struct ata_taskfile tf;
4bd00f6a
TH		 1128 int rc;
1129
1130 DPRINTK("ENTER\n");
1da177e4 1131
4447d351 1132 ahci_stop_engine(ap);
4296971d
TH		 1133
1134 /* clear D2H reception area to properly wait for D2H FIS */
cc0680a5 1135 ata_tf_init(link->device, &tf);
dfd7a3db 1136 tf.command = 0x80;
9977126c 1137 ata_tf_to_fis(&tf, 0, 0, d2h_fis);
4296971d 1138
cc0680a5 1139 rc = sata_std_hardreset(link, class, deadline);
4296971d 1140
4447d351 1141 ahci_start_engine(ap);
1da177e4 1142
cc0680a5 1143 if (rc == 0 && ata_link_online(link))
4bd00f6a
TH		 1144 *class = ahci_dev_classify(ap);
1145 if (*class == ATA_DEV_UNKNOWN)
1146 *class = ATA_DEV_NONE;
1da177e4 1147
4bd00f6a
TH		 1148 DPRINTK("EXIT, rc=%d, class=%u\n", rc, *class);
1149 return rc;
1150}
1151
cc0680a5 1152static int ahci_vt8251_hardreset(struct ata_link *link, unsigned int *class,
d4b2bab4 1153 unsigned long deadline)
ad616ffb 1154{
cc0680a5 1155 struct ata_port *ap = link->ap;
da3dbb17 1156 u32 serror;
ad616ffb
TH		 1157 int rc;
1158
1159 DPRINTK("ENTER\n");
1160
4447d351 1161 ahci_stop_engine(ap);
ad616ffb 1162
cc0680a5 1163 rc = sata_link_hardreset(link, sata_ehc_deb_timing(&link->eh_context),
d4b2bab4 1164 deadline);
ad616ffb
TH		 1165
1166 /* vt8251 needs SError cleared for the port to operate */
da3dbb17
TH		 1167 ahci_scr_read(ap, SCR_ERROR, &serror);
1168 ahci_scr_write(ap, SCR_ERROR, serror);
ad616ffb 1169
4447d351 1170 ahci_start_engine(ap);
ad616ffb
TH		 1171
1172 DPRINTK("EXIT, rc=%d, class=%u\n", rc, *class);
1173
1174 /* vt8251 doesn't clear BSY on signature FIS reception,
1175 * request follow-up softreset.
1176 */
1177 return rc ?: -EAGAIN;
1178}
1179
cc0680a5 1180static void ahci_postreset(struct ata_link *link, unsigned int *class)
4bd00f6a 1181{
cc0680a5 1182 struct ata_port *ap = link->ap;
4447d351 1183 void __iomem *port_mmio = ahci_port_base(ap);
4bd00f6a
TH		 1184 u32 new_tmp, tmp;
1185
cc0680a5 1186 ata_std_postreset(link, class);
02eaa666
JG		 1187
1188 /* Make sure port's ATAPI bit is set appropriately */
1189 new_tmp = tmp = readl(port_mmio + PORT_CMD);
4bd00f6a 1190 if (*class == ATA_DEV_ATAPI)
02eaa666
JG		 1191 new_tmp |= PORT_CMD_ATAPI;
1192 else
1193 new_tmp &= ~PORT_CMD_ATAPI;
1194 if (new_tmp != tmp) {
1195 writel(new_tmp, port_mmio + PORT_CMD);
1196 readl(port_mmio + PORT_CMD); /* flush */
1197 }
1da177e4
LT		 1198}
1199
1200static u8 ahci_check_status(struct ata_port *ap)
1201{
0d5ff566 1202 void __iomem *mmio = ap->ioaddr.cmd_addr;
1da177e4
LT		 1203
1204 return readl(mmio + PORT_TFDATA) & 0xFF;
1205}
1206
1da177e4
LT		 1207static void ahci_tf_read(struct ata_port *ap, struct ata_taskfile *tf)
1208{
1209 struct ahci_port_priv *pp = ap->private_data;
1210 u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
1211
1212 ata_tf_from_fis(d2h_fis, tf);
1213}
1214
12fad3f9 1215static unsigned int ahci_fill_sg(struct ata_queued_cmd *qc, void *cmd_tbl)
1da177e4 1216{
cedc9a47
JG		 1217 struct scatterlist *sg;
1218 struct ahci_sg *ahci_sg;
828d09de 1219 unsigned int n_sg = 0;
1da177e4
LT		 1220
1221 VPRINTK("ENTER\n");
1222
1223 /*
1224 * Next, the S/G list.
1225 */
12fad3f9 1226 ahci_sg = cmd_tbl + AHCI_CMD_TBL_HDR_SZ;
cedc9a47
JG		 1227 ata_for_each_sg(sg, qc) {
1228 dma_addr_t addr = sg_dma_address(sg);
1229 u32 sg_len = sg_dma_len(sg);
1230
1231 ahci_sg->addr = cpu_to_le32(addr & 0xffffffff);
1232 ahci_sg->addr_hi = cpu_to_le32((addr >> 16) >> 16);
1233 ahci_sg->flags_size = cpu_to_le32(sg_len - 1);
828d09de 1234
cedc9a47 1235 ahci_sg++;
828d09de 1236 n_sg++;
1da177e4 1237 }
828d09de
JG		 1238
1239 return n_sg;
1da177e4
LT		 1240}
1241
1242static void ahci_qc_prep(struct ata_queued_cmd *qc)
1243{
a0ea7328
JG		 1244 struct ata_port *ap = qc->ap;
1245 struct ahci_port_priv *pp = ap->private_data;
cc9278ed 1246 int is_atapi = is_atapi_taskfile(&qc->tf);
12fad3f9 1247 void *cmd_tbl;
1da177e4
LT		 1248 u32 opts;
1249 const u32 cmd_fis_len = 5; /* five dwords */
828d09de 1250 unsigned int n_elem;
1da177e4 1251
1da177e4
LT		 1252 /*
1253 * Fill in command table information. First, the header,
1254 * a SATA Register - Host to Device command FIS.
1255 */
12fad3f9
TH		 1256 cmd_tbl = pp->cmd_tbl + qc->tag * AHCI_CMD_TBL_SZ;
1257
9977126c 1258 ata_tf_to_fis(&qc->tf, 0, 1, cmd_tbl);
cc9278ed 1259 if (is_atapi) {
12fad3f9
TH		 1260 memset(cmd_tbl + AHCI_CMD_TBL_CDB, 0, 32);
1261 memcpy(cmd_tbl + AHCI_CMD_TBL_CDB, qc->cdb, qc->dev->cdb_len);
a0ea7328 1262 }
1da177e4 1263
cc9278ed
TH		 1264 n_elem = 0;
1265 if (qc->flags & ATA_QCFLAG_DMAMAP)
12fad3f9 1266 n_elem = ahci_fill_sg(qc, cmd_tbl);
1da177e4 1267
cc9278ed
TH		 1268 /*
1269 * Fill in command slot information.
1270 */
1271 opts = cmd_fis_len | n_elem << 16;
1272 if (qc->tf.flags & ATA_TFLAG_WRITE)
1273 opts |= AHCI_CMD_WRITE;
1274 if (is_atapi)
4b10e559 1275 opts |= AHCI_CMD_ATAPI | AHCI_CMD_PREFETCH;
828d09de 1276
12fad3f9 1277 ahci_fill_cmd_slot(pp, qc->tag, opts);
1da177e4
LT		 1278}
1279
78cd52d0 1280static void ahci_error_intr(struct ata_port *ap, u32 irq_stat)
1da177e4 1281{
78cd52d0 1282 struct ahci_port_priv *pp = ap->private_data;
9af5c9c9 1283 struct ata_eh_info *ehi = &ap->link.eh_info;
78cd52d0
TH		 1284 unsigned int err_mask = 0, action = 0;
1285 struct ata_queued_cmd *qc;
1286 u32 serror;
1da177e4 1287
78cd52d0 1288 ata_ehi_clear_desc(ehi);
1da177e4 1289
78cd52d0 1290 /* AHCI needs SError cleared; otherwise, it might lock up */
da3dbb17 1291 ahci_scr_read(ap, SCR_ERROR, &serror);
78cd52d0 1292 ahci_scr_write(ap, SCR_ERROR, serror);
1da177e4 1293
78cd52d0
TH		 1294 /* analyze @irq_stat */
1295 ata_ehi_push_desc(ehi, "irq_stat 0x%08x", irq_stat);
1296
41669553
TH		 1297 /* some controllers set IRQ_IF_ERR on device errors, ignore it */
1298 if (ap->flags & AHCI_FLAG_IGN_IRQ_IF_ERR)
1299 irq_stat &= ~PORT_IRQ_IF_ERR;
1300
55a61604 1301 if (irq_stat & PORT_IRQ_TF_ERR) {
78cd52d0 1302 err_mask |= AC_ERR_DEV;
55a61604
CH		 1303 if (ap->flags & AHCI_FLAG_IGN_SERR_INTERNAL)
1304 serror &= ~SERR_INTERNAL;
1305 }
78cd52d0
TH		 1306
1307 if (irq_stat & (PORT_IRQ_HBUS_ERR | PORT_IRQ_HBUS_DATA_ERR)) {
1308 err_mask |= AC_ERR_HOST_BUS;
1309 action |= ATA_EH_SOFTRESET;
1da177e4
LT		 1310 }
1311
78cd52d0
TH		 1312 if (irq_stat & PORT_IRQ_IF_ERR) {
1313 err_mask |= AC_ERR_ATA_BUS;
1314 action |= ATA_EH_SOFTRESET;
b64bbc39 1315 ata_ehi_push_desc(ehi, "interface fatal error");
78cd52d0 1316 }
1da177e4 1317
78cd52d0 1318 if (irq_stat & (PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY)) {
4296971d 1319 ata_ehi_hotplugged(ehi);
b64bbc39 1320 ata_ehi_push_desc(ehi, "%s", irq_stat & PORT_IRQ_CONNECT ?
78cd52d0
TH		 1321 "connection status changed" : "PHY RDY changed");
1322 }
1323
1324 if (irq_stat & PORT_IRQ_UNK_FIS) {
1325 u32 *unk = (u32 *)(pp->rx_fis + RX_FIS_UNK);
1da177e4 1326
78cd52d0
TH		 1327 err_mask |= AC_ERR_HSM;
1328 action |= ATA_EH_SOFTRESET;
b64bbc39 1329 ata_ehi_push_desc(ehi, "unknown FIS %08x %08x %08x %08x",
78cd52d0
TH		 1330 unk[0], unk[1], unk[2], unk[3]);
1331 }
1da177e4 1332
78cd52d0
TH		 1333 /* okay, let's hand over to EH */
1334 ehi->serror |= serror;
1335 ehi->action |= action;
b8f6153e 1336
9af5c9c9 1337 qc = ata_qc_from_tag(ap, ap->link.active_tag);
78cd52d0
TH		 1338 if (qc)
1339 qc->err_mask |= err_mask;
1340 else
1341 ehi->err_mask |= err_mask;
a72ec4ce 1342
78cd52d0
TH		 1343 if (irq_stat & PORT_IRQ_FREEZE)
1344 ata_port_freeze(ap);
1345 else
1346 ata_port_abort(ap);
1da177e4
LT		 1347}
1348
df69c9c5 1349static void ahci_port_intr(struct ata_port *ap)
1da177e4 1350{
4447d351 1351 void __iomem *port_mmio = ap->ioaddr.cmd_addr;
9af5c9c9 1352 struct ata_eh_info *ehi = &ap->link.eh_info;
0291f95f 1353 struct ahci_port_priv *pp = ap->private_data;
12fad3f9 1354 u32 status, qc_active;
0291f95f 1355 int rc, known_irq = 0;
1da177e4
LT		 1356
1357 status = readl(port_mmio + PORT_IRQ_STAT);
1358 writel(status, port_mmio + PORT_IRQ_STAT);
1359
78cd52d0
TH		 1360 if (unlikely(status & PORT_IRQ_ERROR)) {
1361 ahci_error_intr(ap, status);
1362 return;
1da177e4
LT		 1363 }
1364
2f294968
KCA		 1365 if (status & PORT_IRQ_SDB_FIS) {
1366 /*
1367 * if this is an ATAPI device with AN turned on,
1368 * then we should interrogate the device to
1369 * determine the cause of the interrupt
1370 *
1371 * for AN - this we should check the SDB FIS
1372 * and find the I and N bits set
1373 */
1374 const __le32 *f = pp->rx_fis + RX_FIS_SDB;
1375 u32 f0 = le32_to_cpu(f[0]);
1376
1377 /* check the 'N' bit in word 0 of the FIS */
1378 if (f0 & (1 << 15)) {
1379 int port_addr = ((f0 & 0x00000f00) >> 8);
1380 struct ata_device *adev;
1381 if (port_addr < ATA_MAX_DEVICES) {
1382 adev = &ap->link.device[port_addr];
1383 if (adev->flags & ATA_DFLAG_AN)
1384 ata_scsi_media_change_notify(adev);
1385 }
1386 }
1387 }
1388
9af5c9c9 1389 if (ap->link.sactive)
12fad3f9
TH		 1390 qc_active = readl(port_mmio + PORT_SCR_ACT);
1391 else
1392 qc_active = readl(port_mmio + PORT_CMD_ISSUE);
1393
1394 rc = ata_qc_complete_multiple(ap, qc_active, NULL);
1395 if (rc > 0)
1396 return;
1397 if (rc < 0) {
1398 ehi->err_mask |= AC_ERR_HSM;
1399 ehi->action |= ATA_EH_SOFTRESET;
1400 ata_port_freeze(ap);
1401 return;
1da177e4
LT		 1402 }
1403
2a3917a8
TH		 1404 /* hmmm... a spurious interupt */
1405
0291f95f
TH		 1406 /* if !NCQ, ignore. No modern ATA device has broken HSM
1407 * implementation for non-NCQ commands.
1408 */
9af5c9c9 1409 if (!ap->link.sactive)
12fad3f9
TH		 1410 return;
1411
0291f95f
TH		 1412 if (status & PORT_IRQ_D2H_REG_FIS) {
1413 if (!pp->ncq_saw_d2h)
1414 ata_port_printk(ap, KERN_INFO,
1415 "D2H reg with I during NCQ, "
1416 "this message won't be printed again\n");
1417 pp->ncq_saw_d2h = 1;
1418 known_irq = 1;
1419 }
1420
1421 if (status & PORT_IRQ_DMAS_FIS) {
1422 if (!pp->ncq_saw_dmas)
1423 ata_port_printk(ap, KERN_INFO,
1424 "DMAS FIS during NCQ, "
1425 "this message won't be printed again\n");
1426 pp->ncq_saw_dmas = 1;
1427 known_irq = 1;
1428 }
1429
a2bbd0c9 1430 if (status & PORT_IRQ_SDB_FIS) {
04d4f7a1 1431 const __le32 *f = pp->rx_fis + RX_FIS_SDB;
0291f95f 1432
afb2d552
TH		 1433 if (le32_to_cpu(f[1])) {
1434 /* SDB FIS containing spurious completions
1435 * might be dangerous, whine and fail commands
1436 * with HSM violation. EH will turn off NCQ
1437 * after several such failures.
1438 */
1439 ata_ehi_push_desc(ehi,
1440 "spurious completions during NCQ "
1441 "issue=0x%x SAct=0x%x FIS=%08x:%08x",
1442 readl(port_mmio + PORT_CMD_ISSUE),
1443 readl(port_mmio + PORT_SCR_ACT),
1444 le32_to_cpu(f[0]), le32_to_cpu(f[1]));
1445 ehi->err_mask |= AC_ERR_HSM;
1446 ehi->action |= ATA_EH_SOFTRESET;
1447 ata_port_freeze(ap);
1448 } else {
1449 if (!pp->ncq_saw_sdb)
1450 ata_port_printk(ap, KERN_INFO,
1451 "spurious SDB FIS %08x:%08x during NCQ, "
1452 "this message won't be printed again\n",
1453 le32_to_cpu(f[0]), le32_to_cpu(f[1]));
1454 pp->ncq_saw_sdb = 1;
1455 }
0291f95f
TH		 1456 known_irq = 1;
1457 }
2a3917a8 1458
0291f95f 1459 if (!known_irq)
78cd52d0 1460 ata_port_printk(ap, KERN_INFO, "spurious interrupt "
0291f95f 1461 "(irq_stat 0x%x active_tag 0x%x sactive 0x%x)\n",
9af5c9c9 1462 status, ap->link.active_tag, ap->link.sactive);
1da177e4
LT		 1463}
1464
1465static void ahci_irq_clear(struct ata_port *ap)
1466{
1467 /* TODO */
1468}
1469
7d12e780 1470static irqreturn_t ahci_interrupt(int irq, void *dev_instance)
1da177e4 1471{
cca3974e 1472 struct ata_host *host = dev_instance;
1da177e4
LT		 1473 struct ahci_host_priv *hpriv;
1474 unsigned int i, handled = 0;
ea6ba10b 1475 void __iomem *mmio;
1da177e4
LT		 1476 u32 irq_stat, irq_ack = 0;
1477
1478 VPRINTK("ENTER\n");
1479
cca3974e 1480 hpriv = host->private_data;
0d5ff566 1481 mmio = host->iomap[AHCI_PCI_BAR];
1da177e4
LT		 1482
1483 /* sigh. 0xffffffff is a valid return from h/w */
1484 irq_stat = readl(mmio + HOST_IRQ_STAT);
1485 irq_stat &= hpriv->port_map;
1486 if (!irq_stat)
1487 return IRQ_NONE;
1488
cca3974e 1489 spin_lock(&host->lock);
1da177e4 1490
cca3974e 1491 for (i = 0; i < host->n_ports; i++) {
1da177e4 1492 struct ata_port *ap;
1da177e4 1493
67846b30
JG		 1494 if (!(irq_stat & (1 << i)))
1495 continue;
1496
cca3974e 1497 ap = host->ports[i];
67846b30 1498 if (ap) {
df69c9c5 1499 ahci_port_intr(ap);
67846b30
JG		 1500 VPRINTK("port %u\n", i);
1501 } else {
1502 VPRINTK("port %u (no irq)\n", i);
6971ed1f 1503 if (ata_ratelimit())
cca3974e 1504 dev_printk(KERN_WARNING, host->dev,
a9524a76 1505 "interrupt on disabled port %u\n", i);
1da177e4 1506 }
67846b30
JG		 1507
1508 irq_ack |= (1 << i);
1da177e4
LT		 1509 }
1510
1511 if (irq_ack) {
1512 writel(irq_ack, mmio + HOST_IRQ_STAT);
1513 handled = 1;
1514 }
1515
cca3974e 1516 spin_unlock(&host->lock);
1da177e4
LT		 1517
1518 VPRINTK("EXIT\n");
1519
1520 return IRQ_RETVAL(handled);
1521}
1522
9a3d9eb0 1523static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc)
1da177e4
LT		 1524{
1525 struct ata_port *ap = qc->ap;
4447d351 1526 void __iomem *port_mmio = ahci_port_base(ap);
1da177e4 1527
12fad3f9
TH		 1528 if (qc->tf.protocol == ATA_PROT_NCQ)
1529 writel(1 << qc->tag, port_mmio + PORT_SCR_ACT);
1530 writel(1 << qc->tag, port_mmio + PORT_CMD_ISSUE);
1da177e4
LT		 1531 readl(port_mmio + PORT_CMD_ISSUE); /* flush */
1532
1533 return 0;
1534}
1535
78cd52d0
TH		 1536static void ahci_freeze(struct ata_port *ap)
1537{
4447d351 1538 void __iomem *port_mmio = ahci_port_base(ap);
78cd52d0
TH		 1539
1540 /* turn IRQ off */
1541 writel(0, port_mmio + PORT_IRQ_MASK);
1542}
1543
1544static void ahci_thaw(struct ata_port *ap)
1545{
0d5ff566 1546 void __iomem *mmio = ap->host->iomap[AHCI_PCI_BAR];
4447d351 1547 void __iomem *port_mmio = ahci_port_base(ap);
78cd52d0 1548 u32 tmp;
a7384925 1549 struct ahci_port_priv *pp = ap->private_data;
78cd52d0
TH		 1550
1551 /* clear IRQ */
1552 tmp = readl(port_mmio + PORT_IRQ_STAT);
1553 writel(tmp, port_mmio + PORT_IRQ_STAT);
a718728f 1554 writel(1 << ap->port_no, mmio + HOST_IRQ_STAT);
78cd52d0
TH		 1555
1556 /* turn IRQ back on */
a7384925 1557 writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
78cd52d0
TH		 1558}
1559
1560static void ahci_error_handler(struct ata_port *ap)
1561{
b51e9e5d 1562 if (!(ap->pflags & ATA_PFLAG_FROZEN)) {
78cd52d0 1563 /* restart engine */
4447d351
TH		 1564 ahci_stop_engine(ap);
1565 ahci_start_engine(ap);
78cd52d0
TH		 1566 }
1567
1568 /* perform recovery */
4aeb0e32 1569 ata_do_eh(ap, ata_std_prereset, ahci_softreset, ahci_hardreset,
f5914a46 1570 ahci_postreset);
78cd52d0
TH		 1571}
1572
ad616ffb
TH		 1573static void ahci_vt8251_error_handler(struct ata_port *ap)
1574{
ad616ffb
TH		 1575 if (!(ap->pflags & ATA_PFLAG_FROZEN)) {
1576 /* restart engine */
4447d351
TH		 1577 ahci_stop_engine(ap);
1578 ahci_start_engine(ap);
ad616ffb
TH		 1579 }
1580
1581 /* perform recovery */
1582 ata_do_eh(ap, ata_std_prereset, ahci_softreset, ahci_vt8251_hardreset,
1583 ahci_postreset);
1584}
1585
78cd52d0
TH		 1586static void ahci_post_internal_cmd(struct ata_queued_cmd *qc)
1587{
1588 struct ata_port *ap = qc->ap;
1589
d2e75dff
TH		 1590 /* make DMA engine forget about the failed command */
1591 if (qc->flags & ATA_QCFLAG_FAILED)
1592 ahci_kick_engine(ap, 1);
78cd52d0
TH		 1593}
1594
028a2596
AD		 1595static int ahci_port_resume(struct ata_port *ap)
1596{
1597 ahci_power_up(ap);
1598 ahci_start_port(ap);
1599
1600 return 0;
1601}
1602
438ac6d5 1603#ifdef CONFIG_PM
c1332875
TH		 1604static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg)
1605{
c1332875
TH		 1606 const char *emsg = NULL;
1607 int rc;
1608
4447d351 1609 rc = ahci_deinit_port(ap, &emsg);
8e16f941 1610 if (rc == 0)
4447d351 1611 ahci_power_down(ap);
8e16f941 1612 else {
c1332875 1613 ata_port_printk(ap, KERN_ERR, "%s (%d)\n", emsg, rc);
df69c9c5 1614 ahci_start_port(ap);
c1332875
TH		 1615 }
1616
1617 return rc;
1618}
1619
c1332875
TH		 1620static int ahci_pci_device_suspend(struct pci_dev *pdev, pm_message_t mesg)
1621{
cca3974e 1622 struct ata_host *host = dev_get_drvdata(&pdev->dev);
0d5ff566 1623 void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
c1332875
TH		 1624 u32 ctl;
1625
1626 if (mesg.event == PM_EVENT_SUSPEND) {
1627 /* AHCI spec rev1.1 section 8.3.3:
1628 * Software must disable interrupts prior to requesting a
1629 * transition of the HBA to D3 state.
1630 */
1631 ctl = readl(mmio + HOST_CTL);
1632 ctl &= ~HOST_IRQ_EN;
1633 writel(ctl, mmio + HOST_CTL);
1634 readl(mmio + HOST_CTL); /* flush */
1635 }
1636
1637 return ata_pci_device_suspend(pdev, mesg);
1638}
1639
1640static int ahci_pci_device_resume(struct pci_dev *pdev)
1641{
cca3974e 1642 struct ata_host *host = dev_get_drvdata(&pdev->dev);
c1332875
TH		 1643 int rc;
1644
553c4aa6
TH		 1645 rc = ata_pci_device_do_resume(pdev);
1646 if (rc)
1647 return rc;
c1332875
TH		 1648
1649 if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
4447d351 1650 rc = ahci_reset_controller(host);
c1332875
TH		 1651 if (rc)
1652 return rc;
1653
4447d351 1654 ahci_init_controller(host);
c1332875
TH		 1655 }
1656
cca3974e 1657 ata_host_resume(host);
c1332875
TH		 1658
1659 return 0;
1660}
438ac6d5 1661#endif
c1332875 1662
254950cd
TH		 1663static int ahci_port_start(struct ata_port *ap)
1664{
cca3974e 1665 struct device *dev = ap->host->dev;
254950cd 1666 struct ahci_port_priv *pp;
254950cd
TH		 1667 void *mem;
1668 dma_addr_t mem_dma;
1669 int rc;
1670
24dc5f33 1671 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
254950cd
TH		 1672 if (!pp)
1673 return -ENOMEM;
254950cd
TH		 1674
1675 rc = ata_pad_alloc(ap, dev);
24dc5f33 1676 if (rc)
254950cd 1677 return rc;
254950cd 1678
24dc5f33
TH		 1679 mem = dmam_alloc_coherent(dev, AHCI_PORT_PRIV_DMA_SZ, &mem_dma,
1680 GFP_KERNEL);
1681 if (!mem)
254950cd 1682 return -ENOMEM;
254950cd
TH		 1683 memset(mem, 0, AHCI_PORT_PRIV_DMA_SZ);
1684
1685 /*
1686 * First item in chunk of DMA memory: 32-slot command table,
1687 * 32 bytes each in size
1688 */
1689 pp->cmd_slot = mem;
1690 pp->cmd_slot_dma = mem_dma;
1691
1692 mem += AHCI_CMD_SLOT_SZ;
1693 mem_dma += AHCI_CMD_SLOT_SZ;
1694
1695 /*
1696 * Second item: Received-FIS area
1697 */
1698 pp->rx_fis = mem;
1699 pp->rx_fis_dma = mem_dma;
1700
1701 mem += AHCI_RX_FIS_SZ;
1702 mem_dma += AHCI_RX_FIS_SZ;
1703
1704 /*
1705 * Third item: data area for storing a single command
1706 * and its scatter-gather table
1707 */
1708 pp->cmd_tbl = mem;
1709 pp->cmd_tbl_dma = mem_dma;
1710
a7384925
KCA		 1711 /*
1712 * Save off initial list of interrupts to be enabled.
1713 * This could be changed later
1714 */
1715 pp->intr_mask = DEF_PORT_IRQ;
1716
254950cd
TH		 1717 ap->private_data = pp;
1718
df69c9c5
JG		 1719 /* engage engines, captain */
1720 return ahci_port_resume(ap);
254950cd
TH		 1721}
1722
1723static void ahci_port_stop(struct ata_port *ap)
1724{
0be0aa98
TH		 1725 const char *emsg = NULL;
1726 int rc;
254950cd 1727
0be0aa98 1728 /* de-initialize port */
4447d351 1729 rc = ahci_deinit_port(ap, &emsg);
0be0aa98
TH		 1730 if (rc)
1731 ata_port_printk(ap, KERN_WARNING, "%s (%d)\n", emsg, rc);
254950cd
TH		 1732}
1733
4447d351 1734static int ahci_configure_dma_masks(struct pci_dev *pdev, int using_dac)
1da177e4 1735{
1da177e4 1736 int rc;
1da177e4 1737
1da177e4
LT		 1738 if (using_dac &&
1739 !pci_set_dma_mask(pdev, DMA_64BIT_MASK)) {
1740 rc = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
1741 if (rc) {
1742 rc = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
1743 if (rc) {
a9524a76
JG		 1744 dev_printk(KERN_ERR, &pdev->dev,
1745 "64-bit DMA enable failed\n");
1da177e4
LT		 1746 return rc;
1747 }
1748 }
1da177e4
LT		 1749 } else {
1750 rc = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
1751 if (rc) {
a9524a76
JG		 1752 dev_printk(KERN_ERR, &pdev->dev,
1753 "32-bit DMA enable failed\n");
1da177e4
LT		 1754 return rc;
1755 }
1756 rc = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
1757 if (rc) {
a9524a76
JG		 1758 dev_printk(KERN_ERR, &pdev->dev,
1759 "32-bit consistent DMA enable failed\n");
1da177e4
LT		 1760 return rc;
1761 }
1762 }
1da177e4
LT		 1763 return 0;
1764}
1765
4447d351 1766static void ahci_print_info(struct ata_host *host)
1da177e4 1767{
4447d351
TH		 1768 struct ahci_host_priv *hpriv = host->private_data;
1769 struct pci_dev *pdev = to_pci_dev(host->dev);
1770 void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
1da177e4
LT		 1771 u32 vers, cap, impl, speed;
1772 const char *speed_s;
1773 u16 cc;
1774 const char *scc_s;
1775
1776 vers = readl(mmio + HOST_VERSION);
1777 cap = hpriv->cap;
1778 impl = hpriv->port_map;
1779
1780 speed = (cap >> 20) & 0xf;
1781 if (speed == 1)
1782 speed_s = "1.5";
1783 else if (speed == 2)
1784 speed_s = "3";
1785 else
1786 speed_s = "?";
1787
1788 pci_read_config_word(pdev, 0x0a, &cc);
c9f89475 1789 if (cc == PCI_CLASS_STORAGE_IDE)
1da177e4 1790 scc_s = "IDE";
c9f89475 1791 else if (cc == PCI_CLASS_STORAGE_SATA)
1da177e4 1792 scc_s = "SATA";
c9f89475 1793 else if (cc == PCI_CLASS_STORAGE_RAID)
1da177e4
LT		 1794 scc_s = "RAID";
1795 else
1796 scc_s = "unknown";
1797
a9524a76
JG		 1798 dev_printk(KERN_INFO, &pdev->dev,
1799 "AHCI %02x%02x.%02x%02x "
1da177e4
LT		 1800 "%u slots %u ports %s Gbps 0x%x impl %s mode\n"
1801 ,
1da177e4
LT		 1802
1803 (vers >> 24) & 0xff,
1804 (vers >> 16) & 0xff,
1805 (vers >> 8) & 0xff,
1806 vers & 0xff,
1807
1808 ((cap >> 8) & 0x1f) + 1,
1809 (cap & 0x1f) + 1,
1810 speed_s,
1811 impl,
1812 scc_s);
1813
a9524a76
JG		 1814 dev_printk(KERN_INFO, &pdev->dev,
1815 "flags: "
203ef6c4
TH		 1816 "%s%s%s%s%s%s%s"
1817 "%s%s%s%s%s%s%s\n"
1da177e4 1818 ,
1da177e4
LT		 1819
1820 cap & (1 << 31) ? "64bit " : "",
1821 cap & (1 << 30) ? "ncq " : "",
203ef6c4 1822 cap & (1 << 29) ? "sntf " : "",
1da177e4
LT		 1823 cap & (1 << 28) ? "ilck " : "",
1824 cap & (1 << 27) ? "stag " : "",
1825 cap & (1 << 26) ? "pm " : "",
1826 cap & (1 << 25) ? "led " : "",
1827
1828 cap & (1 << 24) ? "clo " : "",
1829 cap & (1 << 19) ? "nz " : "",
1830 cap & (1 << 18) ? "only " : "",
1831 cap & (1 << 17) ? "pmp " : "",
1832 cap & (1 << 15) ? "pio " : "",
1833 cap & (1 << 14) ? "slum " : "",
1834 cap & (1 << 13) ? "part " : ""
1835 );
1836}
1837
24dc5f33 1838static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
1da177e4
LT		 1839{
1840 static int printed_version;
4447d351
TH		 1841 struct ata_port_info pi = ahci_port_info[ent->driver_data];
1842 const struct ata_port_info *ppi[] = { &pi, NULL };
24dc5f33 1843 struct device *dev = &pdev->dev;
1da177e4 1844 struct ahci_host_priv *hpriv;
4447d351
TH		 1845 struct ata_host *host;
1846 int i, rc;
1da177e4
LT		 1847
1848 VPRINTK("ENTER\n");
1849
12fad3f9
TH		 1850 WARN_ON(ATA_MAX_QUEUE > AHCI_MAX_CMDS);
1851
1da177e4 1852 if (!printed_version++)
a9524a76 1853 dev_printk(KERN_DEBUG, &pdev->dev, "version " DRV_VERSION "\n");
1da177e4 1854
4447d351 1855 /* acquire resources */
24dc5f33 1856 rc = pcim_enable_device(pdev);
1da177e4
LT		 1857 if (rc)
1858 return rc;
1859
0d5ff566
TH		 1860 rc = pcim_iomap_regions(pdev, 1 << AHCI_PCI_BAR, DRV_NAME);
1861 if (rc == -EBUSY)
24dc5f33 1862 pcim_pin_device(pdev);
0d5ff566 1863 if (rc)
24dc5f33 1864 return rc;
1da177e4 1865
cd70c266 1866 if ((pi.flags & AHCI_FLAG_NO_MSI) || pci_enable_msi(pdev))
907f4678 1867 pci_intx(pdev, 1);
1da177e4 1868
24dc5f33
TH		 1869 hpriv = devm_kzalloc(dev, sizeof(*hpriv), GFP_KERNEL);
1870 if (!hpriv)
1871 return -ENOMEM;
1da177e4 1872
4447d351
TH		 1873 /* save initial config */
1874 ahci_save_initial_config(pdev, &pi, hpriv);
1da177e4 1875
4447d351 1876 /* prepare host */
274c1fde 1877 if (hpriv->cap & HOST_CAP_NCQ)
4447d351 1878 pi.flags |= ATA_FLAG_NCQ;
1da177e4 1879
4447d351
TH		 1880 host = ata_host_alloc_pinfo(&pdev->dev, ppi, fls(hpriv->port_map));
1881 if (!host)
1882 return -ENOMEM;
1883 host->iomap = pcim_iomap_table(pdev);
1884 host->private_data = hpriv;
1885
1886 for (i = 0; i < host->n_ports; i++) {
dab632e8
JG		 1887 struct ata_port *ap = host->ports[i];
1888 void __iomem *port_mmio = ahci_port_base(ap);
4447d351 1889
cbcdd875
TH		 1890 ata_port_pbar_desc(ap, AHCI_PCI_BAR, -1, "abar");
1891 ata_port_pbar_desc(ap, AHCI_PCI_BAR,
1892 0x100 + ap->port_no * 0x80, "port");
1893
dab632e8 1894 /* standard SATA port setup */
203ef6c4 1895 if (hpriv->port_map & (1 << i))
4447d351 1896 ap->ioaddr.cmd_addr = port_mmio;
dab632e8
JG		 1897
1898 /* disabled/not-implemented port */
1899 else
1900 ap->ops = &ata_dummy_port_ops;
4447d351 1901 }
d447df14 1902
4447d351
TH		 1903 /* initialize adapter */
1904 rc = ahci_configure_dma_masks(pdev, hpriv->cap & HOST_CAP_64);
1da177e4 1905 if (rc)
24dc5f33 1906 return rc;
1da177e4 1907
4447d351
TH		 1908 rc = ahci_reset_controller(host);
1909 if (rc)
1910 return rc;
1da177e4 1911
4447d351
TH		 1912 ahci_init_controller(host);
1913 ahci_print_info(host);
1da177e4 1914
4447d351
TH		 1915 pci_set_master(pdev);
1916 return ata_host_activate(host, pdev->irq, ahci_interrupt, IRQF_SHARED,
1917 &ahci_sht);
907f4678 1918}
1da177e4
LT		 1919
1920static int __init ahci_init(void)
1921{
b7887196 1922 return pci_register_driver(&ahci_pci_driver);
1da177e4
LT		 1923}
1924
1da177e4
LT		 1925static void __exit ahci_exit(void)
1926{
1927 pci_unregister_driver(&ahci_pci_driver);
1928}
1929
1930
1931MODULE_AUTHOR("Jeff Garzik");
1932MODULE_DESCRIPTION("AHCI SATA low-level driver");
1933MODULE_LICENSE("GPL");
1934MODULE_DEVICE_TABLE(pci, ahci_pci_tbl);
6885433c 1935MODULE_VERSION(DRV_VERSION);
1da177e4
LT		 1936
1937module_init(ahci_init);
1938module_exit(ahci_exit);
Clone of davem's net-next-2.6 tree