]> bbs.cooldavid.org Git - net-next-2.6.git/blame - drivers/ata/sata_nv.c
git://bbs.cooldavid.org / net-next-2.6.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
[libata] ahci: Restore SB600 SATA controller 64 bit DMA
[net-next-2.6.git] / drivers / ata / sata_nv.c
CommitLineData
1da177e4
LT		 1/*
2 * sata_nv.c - NVIDIA nForce SATA
3 *
4 * Copyright 2004 NVIDIA Corp. All rights reserved.
5 * Copyright 2004 Andrew Chew
6 *
aa7e16d6
JG		 7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2, or (at your option)
11 * any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with this program; see the file COPYING. If not, write to
20 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
1da177e4 21 *
af36d7f0
JG		 22 *
23 * libata documentation is available via 'make {ps|pdf}docs',
24 * as Documentation/DocBook/libata.*
25 *
26 * No hardware documentation available outside of NVIDIA.
27 * This driver programs the NVIDIA SATA controller in a similar
28 * fashion as with other PCI IDE BMDMA controllers, with a few
29 * NV-specific details such as register offsets, SATA phy location,
30 * hotplug info, etc.
31 *
fbbb262d
RH		 32 * CK804/MCP04 controllers support an alternate programming interface
33 * similar to the ADMA specification (with some modifications).
34 * This allows the use of NCQ. Non-DMA-mapped ATA commands are still
35 * sent through the legacy interface.
36 *
1da177e4
LT		 37 */
38
1da177e4
LT		 39#include <linux/kernel.h>
40#include <linux/module.h>
41#include <linux/pci.h>
42#include <linux/init.h>
43#include <linux/blkdev.h>
44#include <linux/delay.h>
45#include <linux/interrupt.h>
a9524a76 46#include <linux/device.h>
1da177e4 47#include <scsi/scsi_host.h>
fbbb262d 48#include <scsi/scsi_device.h>
1da177e4
LT		 49#include <linux/libata.h>
50
51#define DRV_NAME "sata_nv"
2a3103ce 52#define DRV_VERSION "3.5"
fbbb262d
RH		 53
54#define NV_ADMA_DMA_BOUNDARY 0xffffffffUL
1da177e4 55
10ad05df 56enum {
0d5ff566
TH		 57 NV_MMIO_BAR = 5,
58
10ad05df 59 NV_PORTS = 2,
14bdef98
EIB		 60 NV_PIO_MASK = ATA_PIO4,
61 NV_MWDMA_MASK = ATA_MWDMA2,
62 NV_UDMA_MASK = ATA_UDMA6,
10ad05df
JG		 63 NV_PORT0_SCR_REG_OFFSET = 0x00,
64 NV_PORT1_SCR_REG_OFFSET = 0x40,
1da177e4 65
27e4b274 66 /* INT_STATUS/ENABLE */
10ad05df 67 NV_INT_STATUS = 0x10,
10ad05df 68 NV_INT_ENABLE = 0x11,
27e4b274 69 NV_INT_STATUS_CK804 = 0x440,
10ad05df 70 NV_INT_ENABLE_CK804 = 0x441,
1da177e4 71
27e4b274
TH		 72 /* INT_STATUS/ENABLE bits */
73 NV_INT_DEV = 0x01,
74 NV_INT_PM = 0x02,
75 NV_INT_ADDED = 0x04,
76 NV_INT_REMOVED = 0x08,
77
78 NV_INT_PORT_SHIFT = 4, /* each port occupies 4 bits */
79
39f87582 80 NV_INT_ALL = 0x0f,
5a44efff
TH		 81 NV_INT_MASK = NV_INT_DEV |
82 NV_INT_ADDED | NV_INT_REMOVED,
39f87582 83
27e4b274 84 /* INT_CONFIG */
10ad05df
JG		 85 NV_INT_CONFIG = 0x12,
86 NV_INT_CONFIG_METHD = 0x01, // 0 = INT, 1 = SMI
1da177e4 87
10ad05df
JG		 88 // For PCI config register 20
89 NV_MCP_SATA_CFG_20 = 0x50,
90 NV_MCP_SATA_CFG_20_SATA_SPACE_EN = 0x04,
fbbb262d
RH		 91 NV_MCP_SATA_CFG_20_PORT0_EN = (1 << 17),
92 NV_MCP_SATA_CFG_20_PORT1_EN = (1 << 16),
93 NV_MCP_SATA_CFG_20_PORT0_PWB_EN = (1 << 14),
94 NV_MCP_SATA_CFG_20_PORT1_PWB_EN = (1 << 12),
95
96 NV_ADMA_MAX_CPBS = 32,
97 NV_ADMA_CPB_SZ = 128,
98 NV_ADMA_APRD_SZ = 16,
99 NV_ADMA_SGTBL_LEN = (1024 - NV_ADMA_CPB_SZ) /
100 NV_ADMA_APRD_SZ,
101 NV_ADMA_SGTBL_TOTAL_LEN = NV_ADMA_SGTBL_LEN + 5,
102 NV_ADMA_SGTBL_SZ = NV_ADMA_SGTBL_LEN * NV_ADMA_APRD_SZ,
103 NV_ADMA_PORT_PRIV_DMA_SZ = NV_ADMA_MAX_CPBS *
104 (NV_ADMA_CPB_SZ + NV_ADMA_SGTBL_SZ),
105
106 /* BAR5 offset to ADMA general registers */
107 NV_ADMA_GEN = 0x400,
108 NV_ADMA_GEN_CTL = 0x00,
109 NV_ADMA_NOTIFIER_CLEAR = 0x30,
110
111 /* BAR5 offset to ADMA ports */
112 NV_ADMA_PORT = 0x480,
113
114 /* size of ADMA port register space */
115 NV_ADMA_PORT_SIZE = 0x100,
116
117 /* ADMA port registers */
118 NV_ADMA_CTL = 0x40,
119 NV_ADMA_CPB_COUNT = 0x42,
120 NV_ADMA_NEXT_CPB_IDX = 0x43,
121 NV_ADMA_STAT = 0x44,
122 NV_ADMA_CPB_BASE_LOW = 0x48,
123 NV_ADMA_CPB_BASE_HIGH = 0x4C,
124 NV_ADMA_APPEND = 0x50,
125 NV_ADMA_NOTIFIER = 0x68,
126 NV_ADMA_NOTIFIER_ERROR = 0x6C,
127
128 /* NV_ADMA_CTL register bits */
129 NV_ADMA_CTL_HOTPLUG_IEN = (1 << 0),
130 NV_ADMA_CTL_CHANNEL_RESET = (1 << 5),
131 NV_ADMA_CTL_GO = (1 << 7),
132 NV_ADMA_CTL_AIEN = (1 << 8),
133 NV_ADMA_CTL_READ_NON_COHERENT = (1 << 11),
134 NV_ADMA_CTL_WRITE_NON_COHERENT = (1 << 12),
135
136 /* CPB response flag bits */
137 NV_CPB_RESP_DONE = (1 << 0),
138 NV_CPB_RESP_ATA_ERR = (1 << 3),
139 NV_CPB_RESP_CMD_ERR = (1 << 4),
140 NV_CPB_RESP_CPB_ERR = (1 << 7),
141
142 /* CPB control flag bits */
143 NV_CPB_CTL_CPB_VALID = (1 << 0),
144 NV_CPB_CTL_QUEUE = (1 << 1),
145 NV_CPB_CTL_APRD_VALID = (1 << 2),
146 NV_CPB_CTL_IEN = (1 << 3),
147 NV_CPB_CTL_FPDMA = (1 << 4),
148
149 /* APRD flags */
150 NV_APRD_WRITE = (1 << 1),
151 NV_APRD_END = (1 << 2),
152 NV_APRD_CONT = (1 << 3),
153
154 /* NV_ADMA_STAT flags */
155 NV_ADMA_STAT_TIMEOUT = (1 << 0),
156 NV_ADMA_STAT_HOTUNPLUG = (1 << 1),
157 NV_ADMA_STAT_HOTPLUG = (1 << 2),
158 NV_ADMA_STAT_CPBERR = (1 << 4),
159 NV_ADMA_STAT_SERROR = (1 << 5),
160 NV_ADMA_STAT_CMD_COMPLETE = (1 << 6),
161 NV_ADMA_STAT_IDLE = (1 << 8),
162 NV_ADMA_STAT_LEGACY = (1 << 9),
163 NV_ADMA_STAT_STOPPED = (1 << 10),
164 NV_ADMA_STAT_DONE = (1 << 12),
165 NV_ADMA_STAT_ERR = NV_ADMA_STAT_CPBERR |
2dcb407e 166 NV_ADMA_STAT_TIMEOUT,
fbbb262d
RH		 167
168 /* port flags */
169 NV_ADMA_PORT_REGISTER_MODE = (1 << 0),
2dec7555 170 NV_ADMA_ATAPI_SETUP_COMPLETE = (1 << 1),
fbbb262d 171
f140f0f1
KL		 172 /* MCP55 reg offset */
173 NV_CTL_MCP55 = 0x400,
174 NV_INT_STATUS_MCP55 = 0x440,
175 NV_INT_ENABLE_MCP55 = 0x444,
176 NV_NCQ_REG_MCP55 = 0x448,
177
178 /* MCP55 */
179 NV_INT_ALL_MCP55 = 0xffff,
180 NV_INT_PORT_SHIFT_MCP55 = 16, /* each port occupies 16 bits */
181 NV_INT_MASK_MCP55 = NV_INT_ALL_MCP55 & 0xfffd,
182
183 /* SWNCQ ENABLE BITS*/
184 NV_CTL_PRI_SWNCQ = 0x02,
185 NV_CTL_SEC_SWNCQ = 0x04,
186
187 /* SW NCQ status bits*/
188 NV_SWNCQ_IRQ_DEV = (1 << 0),
189 NV_SWNCQ_IRQ_PM = (1 << 1),
190 NV_SWNCQ_IRQ_ADDED = (1 << 2),
191 NV_SWNCQ_IRQ_REMOVED = (1 << 3),
192
193 NV_SWNCQ_IRQ_BACKOUT = (1 << 4),
194 NV_SWNCQ_IRQ_SDBFIS = (1 << 5),
195 NV_SWNCQ_IRQ_DHREGFIS = (1 << 6),
196 NV_SWNCQ_IRQ_DMASETUP = (1 << 7),
197
198 NV_SWNCQ_IRQ_HOTPLUG = NV_SWNCQ_IRQ_ADDED |
199 NV_SWNCQ_IRQ_REMOVED,
200
fbbb262d
RH		 201};
202
203/* ADMA Physical Region Descriptor - one SG segment */
204struct nv_adma_prd {
205 __le64 addr;
206 __le32 len;
207 u8 flags;
208 u8 packet_len;
209 __le16 reserved;
210};
211
212enum nv_adma_regbits {
213 CMDEND = (1 << 15), /* end of command list */
214 WNB = (1 << 14), /* wait-not-BSY */
215 IGN = (1 << 13), /* ignore this entry */
216 CS1n = (1 << (4 + 8)), /* std. PATA signals follow... */
217 DA2 = (1 << (2 + 8)),
218 DA1 = (1 << (1 + 8)),
219 DA0 = (1 << (0 + 8)),
220};
221
222/* ADMA Command Parameter Block
223 The first 5 SG segments are stored inside the Command Parameter Block itself.
224 If there are more than 5 segments the remainder are stored in a separate
225 memory area indicated by next_aprd. */
226struct nv_adma_cpb {
227 u8 resp_flags; /* 0 */
228 u8 reserved1; /* 1 */
229 u8 ctl_flags; /* 2 */
230 /* len is length of taskfile in 64 bit words */
2dcb407e 231 u8 len; /* 3 */
fbbb262d
RH		 232 u8 tag; /* 4 */
233 u8 next_cpb_idx; /* 5 */
234 __le16 reserved2; /* 6-7 */
235 __le16 tf[12]; /* 8-31 */
236 struct nv_adma_prd aprd[5]; /* 32-111 */
237 __le64 next_aprd; /* 112-119 */
238 __le64 reserved3; /* 120-127 */
10ad05df 239};
1da177e4 240
fbbb262d
RH		 241
242struct nv_adma_port_priv {
243 struct nv_adma_cpb *cpb;
244 dma_addr_t cpb_dma;
245 struct nv_adma_prd *aprd;
246 dma_addr_t aprd_dma;
2dcb407e
JG		 247 void __iomem *ctl_block;
248 void __iomem *gen_block;
249 void __iomem *notifier_clear_block;
8959d300 250 u64 adma_dma_mask;
fbbb262d 251 u8 flags;
5e5c74a5 252 int last_issue_ncq;
fbbb262d
RH		 253};
254
cdf56bcf
RH		 255struct nv_host_priv {
256 unsigned long type;
257};
258
f140f0f1
KL		 259struct defer_queue {
260 u32 defer_bits;
261 unsigned int head;
262 unsigned int tail;
263 unsigned int tag[ATA_MAX_QUEUE];
264};
265
266enum ncq_saw_flag_list {
267 ncq_saw_d2h = (1U << 0),
268 ncq_saw_dmas = (1U << 1),
269 ncq_saw_sdb = (1U << 2),
270 ncq_saw_backout = (1U << 3),
271};
272
273struct nv_swncq_port_priv {
274 struct ata_prd *prd; /* our SG list */
275 dma_addr_t prd_dma; /* and its DMA mapping */
276 void __iomem *sactive_block;
277 void __iomem *irq_block;
278 void __iomem *tag_block;
279 u32 qc_active;
280
281 unsigned int last_issue_tag;
282
283 /* fifo circular queue to store deferral command */
284 struct defer_queue defer_queue;
285
286 /* for NCQ interrupt analysis */
287 u32 dhfis_bits;
288 u32 dmafis_bits;
289 u32 sdbfis_bits;
290
291 unsigned int ncq_flags;
292};
293
294
5796d1c4 295#define NV_ADMA_CHECK_INTR(GCTL, PORT) ((GCTL) & (1 << (19 + (12 * (PORT)))))
fbbb262d 296
2dcb407e 297static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
438ac6d5 298#ifdef CONFIG_PM
cdf56bcf 299static int nv_pci_device_resume(struct pci_dev *pdev);
438ac6d5 300#endif
cca3974e 301static void nv_ck804_host_stop(struct ata_host *host);
7d12e780
DH		 302static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance);
303static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance);
304static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance);
82ef04fb
TH		 305static int nv_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val);
306static int nv_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val);
1da177e4 307
e8caa3c7
TH		 308static int nv_noclassify_hardreset(struct ata_link *link, unsigned int *class,
309 unsigned long deadline);
39f87582
TH		 310static void nv_nf2_freeze(struct ata_port *ap);
311static void nv_nf2_thaw(struct ata_port *ap);
312static void nv_ck804_freeze(struct ata_port *ap);
313static void nv_ck804_thaw(struct ata_port *ap);
fbbb262d 314static int nv_adma_slave_config(struct scsi_device *sdev);
2dec7555 315static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc);
fbbb262d
RH		 316static void nv_adma_qc_prep(struct ata_queued_cmd *qc);
317static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc);
318static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance);
319static void nv_adma_irq_clear(struct ata_port *ap);
320static int nv_adma_port_start(struct ata_port *ap);
321static void nv_adma_port_stop(struct ata_port *ap);
438ac6d5 322#ifdef CONFIG_PM
cdf56bcf
RH		 323static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg);
324static int nv_adma_port_resume(struct ata_port *ap);
438ac6d5 325#endif
53014e25
RH		 326static void nv_adma_freeze(struct ata_port *ap);
327static void nv_adma_thaw(struct ata_port *ap);
fbbb262d
RH		 328static void nv_adma_error_handler(struct ata_port *ap);
329static void nv_adma_host_stop(struct ata_host *host);
f5ecac2d 330static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc);
f2fb344b 331static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf);
39f87582 332
f140f0f1
KL		 333static void nv_mcp55_thaw(struct ata_port *ap);
334static void nv_mcp55_freeze(struct ata_port *ap);
335static void nv_swncq_error_handler(struct ata_port *ap);
336static int nv_swncq_slave_config(struct scsi_device *sdev);
337static int nv_swncq_port_start(struct ata_port *ap);
338static void nv_swncq_qc_prep(struct ata_queued_cmd *qc);
339static void nv_swncq_fill_sg(struct ata_queued_cmd *qc);
340static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc);
341static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis);
342static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance);
343#ifdef CONFIG_PM
344static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg);
345static int nv_swncq_port_resume(struct ata_port *ap);
346#endif
347
1da177e4
LT		 348enum nv_host_type
349{
350 GENERIC,
351 NFORCE2,
27e4b274 352 NFORCE3 = NFORCE2, /* NF2 == NF3 as far as sata_nv is concerned */
fbbb262d 353 CK804,
f140f0f1 354 ADMA,
2d775708 355 MCP5x,
f140f0f1 356 SWNCQ,
1da177e4
LT		 357};
358
3b7d697d 359static const struct pci_device_id nv_pci_tbl[] = {
54bb3a94
JG		 360 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE2S_SATA), NFORCE2 },
361 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA), NFORCE3 },
362 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA2), NFORCE3 },
363 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA), CK804 },
364 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA2), CK804 },
365 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA), CK804 },
366 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA2), CK804 },
2d775708
TH		 367 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA), MCP5x },
368 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2), MCP5x },
369 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA), MCP5x },
370 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2), MCP5x },
e2e031eb
KL		 371 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA), GENERIC },
372 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA2), GENERIC },
373 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA3), GENERIC },
2d2744fc
JG		 374
375 { } /* terminate list */
1da177e4
LT		 376};
377
1da177e4
LT		 378static struct pci_driver nv_pci_driver = {
379 .name = DRV_NAME,
380 .id_table = nv_pci_tbl,
381 .probe = nv_init_one,
438ac6d5 382#ifdef CONFIG_PM
cdf56bcf
RH		 383 .suspend = ata_pci_device_suspend,
384 .resume = nv_pci_device_resume,
438ac6d5 385#endif
1daf9ce7 386 .remove = ata_pci_remove_one,
1da177e4
LT		 387};
388
193515d5 389static struct scsi_host_template nv_sht = {
68d1d07b 390 ATA_BMDMA_SHT(DRV_NAME),
1da177e4
LT		 391};
392
fbbb262d 393static struct scsi_host_template nv_adma_sht = {
68d1d07b 394 ATA_NCQ_SHT(DRV_NAME),
fbbb262d 395 .can_queue = NV_ADMA_MAX_CPBS,
fbbb262d 396 .sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN,
fbbb262d
RH		 397 .dma_boundary = NV_ADMA_DMA_BOUNDARY,
398 .slave_configure = nv_adma_slave_config,
fbbb262d
RH		 399};
400
f140f0f1 401static struct scsi_host_template nv_swncq_sht = {
68d1d07b 402 ATA_NCQ_SHT(DRV_NAME),
f140f0f1 403 .can_queue = ATA_MAX_QUEUE,
f140f0f1 404 .sg_tablesize = LIBATA_MAX_PRD,
f140f0f1
KL		 405 .dma_boundary = ATA_DMA_BOUNDARY,
406 .slave_configure = nv_swncq_slave_config,
f140f0f1
KL		 407};
408
4c1eb90a 409static struct ata_port_operations nv_common_ops = {
029cfd6b 410 .inherits = &ata_bmdma_port_ops,
c96f1732 411 .lost_interrupt = ATA_OP_NULL,
1da177e4
LT		 412 .scr_read = nv_scr_read,
413 .scr_write = nv_scr_write,
1da177e4
LT		 414};
415
4c1eb90a
TH		 416/* OSDL bz11195 reports that link doesn't come online after hardreset
417 * on generic nv's and there have been several other similar reports
418 * on linux-ide. Disable hardreset for generic nv's.
419 */
420static struct ata_port_operations nv_generic_ops = {
421 .inherits = &nv_common_ops,
422 .hardreset = ATA_OP_NULL,
423};
424
7dac745b
TH		 425/* nf2 is ripe with hardreset related problems.
426 *
427 * kernel bz#3352 reports nf2/3 controllers can't determine device
428 * signature reliably. The following thread reports detection failure
429 * on cold boot with the standard debouncing timing.
3c324283
TH		 430 *
431 * http://thread.gmane.org/gmane.linux.ide/34098
432 *
7dac745b
TH		 433 * And bz#12176 reports that hardreset simply doesn't work on nf2.
434 * Give up on it and just don't do hardreset.
3c324283 435 */
029cfd6b 436static struct ata_port_operations nv_nf2_ops = {
7dac745b 437 .inherits = &nv_generic_ops,
39f87582
TH		 438 .freeze = nv_nf2_freeze,
439 .thaw = nv_nf2_thaw,
ada364e8
TH		 440};
441
8d993eaa
TH		 442/* For initial probing after boot and hot plugging, hardreset mostly
443 * works fine on CK804 but curiously, reprobing on the initial port by
444 * rescanning or rmmod/insmod fails to acquire the initial D2H Reg FIS
445 * in somewhat undeterministic way. Use noclassify hardreset.
446 */
029cfd6b 447static struct ata_port_operations nv_ck804_ops = {
4c1eb90a 448 .inherits = &nv_common_ops,
39f87582
TH		 449 .freeze = nv_ck804_freeze,
450 .thaw = nv_ck804_thaw,
8d993eaa 451 .hardreset = nv_noclassify_hardreset,
ada364e8
TH		 452 .host_stop = nv_ck804_host_stop,
453};
454
029cfd6b 455static struct ata_port_operations nv_adma_ops = {
3c324283 456 .inherits = &nv_ck804_ops,
029cfd6b 457
2dec7555 458 .check_atapi_dma = nv_adma_check_atapi_dma,
5682ed33 459 .sff_tf_read = nv_adma_tf_read,
31cc23b3 460 .qc_defer = ata_std_qc_defer,
fbbb262d
RH		 461 .qc_prep = nv_adma_qc_prep,
462 .qc_issue = nv_adma_qc_issue,
5682ed33 463 .sff_irq_clear = nv_adma_irq_clear,
029cfd6b 464
53014e25
RH		 465 .freeze = nv_adma_freeze,
466 .thaw = nv_adma_thaw,
fbbb262d 467 .error_handler = nv_adma_error_handler,
f5ecac2d 468 .post_internal_cmd = nv_adma_post_internal_cmd,
029cfd6b 469
fbbb262d
RH		 470 .port_start = nv_adma_port_start,
471 .port_stop = nv_adma_port_stop,
438ac6d5 472#ifdef CONFIG_PM
cdf56bcf
RH		 473 .port_suspend = nv_adma_port_suspend,
474 .port_resume = nv_adma_port_resume,
438ac6d5 475#endif
fbbb262d
RH		 476 .host_stop = nv_adma_host_stop,
477};
478
2d775708
TH		 479/* Kernel bz#12351 reports that when SWNCQ is enabled, for hotplug to
480 * work, hardreset should be used and hardreset can't report proper
481 * signature, which suggests that mcp5x is closer to nf2 as long as
482 * reset quirkiness is concerned. Define separate ops for mcp5x with
483 * nv_noclassify_hardreset().
484 */
485static struct ata_port_operations nv_mcp5x_ops = {
486 .inherits = &nv_common_ops,
487 .hardreset = nv_noclassify_hardreset,
488};
489
029cfd6b 490static struct ata_port_operations nv_swncq_ops = {
2d775708 491 .inherits = &nv_mcp5x_ops,
029cfd6b 492
f140f0f1
KL		 493 .qc_defer = ata_std_qc_defer,
494 .qc_prep = nv_swncq_qc_prep,
495 .qc_issue = nv_swncq_qc_issue,
029cfd6b 496
f140f0f1
KL		 497 .freeze = nv_mcp55_freeze,
498 .thaw = nv_mcp55_thaw,
499 .error_handler = nv_swncq_error_handler,
029cfd6b 500
f140f0f1
KL		 501#ifdef CONFIG_PM
502 .port_suspend = nv_swncq_port_suspend,
503 .port_resume = nv_swncq_port_resume,
504#endif
505 .port_start = nv_swncq_port_start,
506};
507
95947193
TH		 508struct nv_pi_priv {
509 irq_handler_t irq_handler;
510 struct scsi_host_template *sht;
511};
512
513#define NV_PI_PRIV(_irq_handler, _sht) \
514 &(struct nv_pi_priv){ .irq_handler = _irq_handler, .sht = _sht }
515
1626aeb8 516static const struct ata_port_info nv_port_info[] = {
ada364e8
TH		 517 /* generic */
518 {
0c88758b 519 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
ada364e8
TH		 520 .pio_mask = NV_PIO_MASK,
521 .mwdma_mask = NV_MWDMA_MASK,
522 .udma_mask = NV_UDMA_MASK,
523 .port_ops = &nv_generic_ops,
95947193 524 .private_data = NV_PI_PRIV(nv_generic_interrupt, &nv_sht),
ada364e8
TH		 525 },
526 /* nforce2/3 */
527 {
0c88758b 528 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
ada364e8
TH		 529 .pio_mask = NV_PIO_MASK,
530 .mwdma_mask = NV_MWDMA_MASK,
531 .udma_mask = NV_UDMA_MASK,
532 .port_ops = &nv_nf2_ops,
95947193 533 .private_data = NV_PI_PRIV(nv_nf2_interrupt, &nv_sht),
ada364e8
TH		 534 },
535 /* ck804 */
536 {
0c88758b 537 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
ada364e8
TH		 538 .pio_mask = NV_PIO_MASK,
539 .mwdma_mask = NV_MWDMA_MASK,
540 .udma_mask = NV_UDMA_MASK,
541 .port_ops = &nv_ck804_ops,
95947193 542 .private_data = NV_PI_PRIV(nv_ck804_interrupt, &nv_sht),
ada364e8 543 },
fbbb262d
RH		 544 /* ADMA */
545 {
fbbb262d
RH		 546 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
547 ATA_FLAG_MMIO | ATA_FLAG_NCQ,
548 .pio_mask = NV_PIO_MASK,
549 .mwdma_mask = NV_MWDMA_MASK,
550 .udma_mask = NV_UDMA_MASK,
551 .port_ops = &nv_adma_ops,
95947193 552 .private_data = NV_PI_PRIV(nv_adma_interrupt, &nv_adma_sht),
fbbb262d 553 },
2d775708
TH		 554 /* MCP5x */
555 {
556 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
557 .pio_mask = NV_PIO_MASK,
558 .mwdma_mask = NV_MWDMA_MASK,
559 .udma_mask = NV_UDMA_MASK,
560 .port_ops = &nv_mcp5x_ops,
561 .private_data = NV_PI_PRIV(nv_generic_interrupt, &nv_sht),
562 },
f140f0f1
KL		 563 /* SWNCQ */
564 {
f140f0f1
KL		 565 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
566 ATA_FLAG_NCQ,
f140f0f1
KL		 567 .pio_mask = NV_PIO_MASK,
568 .mwdma_mask = NV_MWDMA_MASK,
569 .udma_mask = NV_UDMA_MASK,
570 .port_ops = &nv_swncq_ops,
95947193 571 .private_data = NV_PI_PRIV(nv_swncq_interrupt, &nv_swncq_sht),
f140f0f1 572 },
1da177e4
LT		 573};
574
575MODULE_AUTHOR("NVIDIA");
576MODULE_DESCRIPTION("low-level driver for NVIDIA nForce SATA controller");
577MODULE_LICENSE("GPL");
578MODULE_DEVICE_TABLE(pci, nv_pci_tbl);
579MODULE_VERSION(DRV_VERSION);
580
06993d22 581static int adma_enabled;
d21279f4 582static int swncq_enabled = 1;
fbbb262d 583
2dec7555
RH		 584static void nv_adma_register_mode(struct ata_port *ap)
585{
2dec7555 586 struct nv_adma_port_priv *pp = ap->private_data;
cdf56bcf 587 void __iomem *mmio = pp->ctl_block;
a2cfe81a
RH		 588 u16 tmp, status;
589 int count = 0;
2dec7555
RH		 590
591 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
592 return;
593
a2cfe81a 594 status = readw(mmio + NV_ADMA_STAT);
2dcb407e 595 while (!(status & NV_ADMA_STAT_IDLE) && count < 20) {
a2cfe81a
RH		 596 ndelay(50);
597 status = readw(mmio + NV_ADMA_STAT);
598 count++;
599 }
2dcb407e 600 if (count == 20)
a2cfe81a
RH		 601 ata_port_printk(ap, KERN_WARNING,
602 "timeout waiting for ADMA IDLE, stat=0x%hx\n",
603 status);
604
2dec7555
RH		 605 tmp = readw(mmio + NV_ADMA_CTL);
606 writew(tmp & ~NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
607
a2cfe81a
RH		 608 count = 0;
609 status = readw(mmio + NV_ADMA_STAT);
2dcb407e 610 while (!(status & NV_ADMA_STAT_LEGACY) && count < 20) {
a2cfe81a
RH		 611 ndelay(50);
612 status = readw(mmio + NV_ADMA_STAT);
613 count++;
614 }
2dcb407e 615 if (count == 20)
a2cfe81a
RH		 616 ata_port_printk(ap, KERN_WARNING,
617 "timeout waiting for ADMA LEGACY, stat=0x%hx\n",
618 status);
619
2dec7555
RH		 620 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
621}
622
623static void nv_adma_mode(struct ata_port *ap)
624{
2dec7555 625 struct nv_adma_port_priv *pp = ap->private_data;
cdf56bcf 626 void __iomem *mmio = pp->ctl_block;
a2cfe81a
RH		 627 u16 tmp, status;
628 int count = 0;
2dec7555
RH		 629
630 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE))
631 return;
f20b16ff 632
2dec7555
RH		 633 WARN_ON(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
634
635 tmp = readw(mmio + NV_ADMA_CTL);
636 writew(tmp | NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
637
a2cfe81a 638 status = readw(mmio + NV_ADMA_STAT);
2dcb407e 639 while (((status & NV_ADMA_STAT_LEGACY) ||
a2cfe81a
RH		 640 !(status & NV_ADMA_STAT_IDLE)) && count < 20) {
641 ndelay(50);
642 status = readw(mmio + NV_ADMA_STAT);
643 count++;
644 }
2dcb407e 645 if (count == 20)
a2cfe81a
RH		 646 ata_port_printk(ap, KERN_WARNING,
647 "timeout waiting for ADMA LEGACY clear and IDLE, stat=0x%hx\n",
648 status);
649
2dec7555
RH		 650 pp->flags &= ~NV_ADMA_PORT_REGISTER_MODE;
651}
652
fbbb262d
RH		 653static int nv_adma_slave_config(struct scsi_device *sdev)
654{
655 struct ata_port *ap = ata_shost_to_port(sdev->host);
2dec7555 656 struct nv_adma_port_priv *pp = ap->private_data;
8959d300
RH		 657 struct nv_adma_port_priv *port0, *port1;
658 struct scsi_device *sdev0, *sdev1;
2dec7555 659 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
8959d300 660 unsigned long segment_boundary, flags;
fbbb262d
RH		 661 unsigned short sg_tablesize;
662 int rc;
2dec7555
RH		 663 int adma_enable;
664 u32 current_reg, new_reg, config_mask;
fbbb262d
RH		 665
666 rc = ata_scsi_slave_config(sdev);
667
668 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
669 /* Not a proper libata device, ignore */
670 return rc;
671
8959d300
RH		 672 spin_lock_irqsave(ap->lock, flags);
673
9af5c9c9 674 if (ap->link.device[sdev->id].class == ATA_DEV_ATAPI) {
fbbb262d
RH		 675 /*
676 * NVIDIA reports that ADMA mode does not support ATAPI commands.
677 * Therefore ATAPI commands are sent through the legacy interface.
678 * However, the legacy interface only supports 32-bit DMA.
679 * Restrict DMA parameters as required by the legacy interface
680 * when an ATAPI device is connected.
681 */
fbbb262d
RH		 682 segment_boundary = ATA_DMA_BOUNDARY;
683 /* Subtract 1 since an extra entry may be needed for padding, see
684 libata-scsi.c */
685 sg_tablesize = LIBATA_MAX_PRD - 1;
f20b16ff 686
2dec7555
RH		 687 /* Since the legacy DMA engine is in use, we need to disable ADMA
688 on the port. */
689 adma_enable = 0;
690 nv_adma_register_mode(ap);
2dcb407e 691 } else {
fbbb262d
RH		 692 segment_boundary = NV_ADMA_DMA_BOUNDARY;
693 sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN;
2dec7555 694 adma_enable = 1;
fbbb262d 695 }
f20b16ff 696
2dec7555
RH		 697 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &current_reg);
698
2dcb407e 699 if (ap->port_no == 1)
2dec7555
RH		 700 config_mask = NV_MCP_SATA_CFG_20_PORT1_EN |
701 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
702 else
703 config_mask = NV_MCP_SATA_CFG_20_PORT0_EN |
704 NV_MCP_SATA_CFG_20_PORT0_PWB_EN;
f20b16ff 705
2dcb407e 706 if (adma_enable) {
2dec7555
RH		 707 new_reg = current_reg | config_mask;
708 pp->flags &= ~NV_ADMA_ATAPI_SETUP_COMPLETE;
2dcb407e 709 } else {
2dec7555
RH		 710 new_reg = current_reg & ~config_mask;
711 pp->flags |= NV_ADMA_ATAPI_SETUP_COMPLETE;
712 }
f20b16ff 713
2dcb407e 714 if (current_reg != new_reg)
2dec7555 715 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, new_reg);
f20b16ff 716
8959d300
RH		 717 port0 = ap->host->ports[0]->private_data;
718 port1 = ap->host->ports[1]->private_data;
719 sdev0 = ap->host->ports[0]->link.device[0].sdev;
720 sdev1 = ap->host->ports[1]->link.device[0].sdev;
721 if ((port0->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
722 (port1->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)) {
723 /** We have to set the DMA mask to 32-bit if either port is in
724 ATAPI mode, since they are on the same PCI device which is
725 used for DMA mapping. If we set the mask we also need to set
726 the bounce limit on both ports to ensure that the block
727 layer doesn't feed addresses that cause DMA mapping to
728 choke. If either SCSI device is not allocated yet, it's OK
729 since that port will discover its correct setting when it
730 does get allocated.
731 Note: Setting 32-bit mask should not fail. */
732 if (sdev0)
733 blk_queue_bounce_limit(sdev0->request_queue,
734 ATA_DMA_MASK);
735 if (sdev1)
736 blk_queue_bounce_limit(sdev1->request_queue,
737 ATA_DMA_MASK);
738
739 pci_set_dma_mask(pdev, ATA_DMA_MASK);
740 } else {
741 /** This shouldn't fail as it was set to this value before */
742 pci_set_dma_mask(pdev, pp->adma_dma_mask);
743 if (sdev0)
744 blk_queue_bounce_limit(sdev0->request_queue,
745 pp->adma_dma_mask);
746 if (sdev1)
747 blk_queue_bounce_limit(sdev1->request_queue,
748 pp->adma_dma_mask);
749 }
750
fbbb262d
RH		 751 blk_queue_segment_boundary(sdev->request_queue, segment_boundary);
752 blk_queue_max_hw_segments(sdev->request_queue, sg_tablesize);
753 ata_port_printk(ap, KERN_INFO,
8959d300
RH		 754 "DMA mask 0x%llX, segment boundary 0x%lX, hw segs %hu\n",
755 (unsigned long long)*ap->host->dev->dma_mask,
756 segment_boundary, sg_tablesize);
757
758 spin_unlock_irqrestore(ap->lock, flags);
759
fbbb262d
RH		 760 return rc;
761}
762
2dec7555
RH		 763static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc)
764{
765 struct nv_adma_port_priv *pp = qc->ap->private_data;
766 return !(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
767}
768
f2fb344b
RH		 769static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf)
770{
3f3debdb
RH		 771 /* Other than when internal or pass-through commands are executed,
772 the only time this function will be called in ADMA mode will be
773 if a command fails. In the failure case we don't care about going
774 into register mode with ADMA commands pending, as the commands will
775 all shortly be aborted anyway. We assume that NCQ commands are not
776 issued via passthrough, which is the only way that switching into
777 ADMA mode could abort outstanding commands. */
f2fb344b
RH		 778 nv_adma_register_mode(ap);
779
9363c382 780 ata_sff_tf_read(ap, tf);
f2fb344b
RH		 781}
782
2dec7555 783static unsigned int nv_adma_tf_to_cpb(struct ata_taskfile *tf, __le16 *cpb)
fbbb262d
RH		 784{
785 unsigned int idx = 0;
786
2dcb407e 787 if (tf->flags & ATA_TFLAG_ISADDR) {
ac3d6b86
RH		 788 if (tf->flags & ATA_TFLAG_LBA48) {
789 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->hob_feature | WNB);
790 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->hob_nsect);
791 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->hob_lbal);
792 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->hob_lbam);
793 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->hob_lbah);
794 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature);
795 } else
796 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature | WNB);
a84471fe 797
ac3d6b86
RH		 798 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->nsect);
799 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->lbal);
800 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->lbam);
801 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->lbah);
fbbb262d 802 }
a84471fe 803
2dcb407e 804 if (tf->flags & ATA_TFLAG_DEVICE)
ac3d6b86 805 cpb[idx++] = cpu_to_le16((ATA_REG_DEVICE << 8) | tf->device);
fbbb262d
RH		 806
807 cpb[idx++] = cpu_to_le16((ATA_REG_CMD << 8) | tf->command | CMDEND);
a84471fe 808
2dcb407e 809 while (idx < 12)
ac3d6b86 810 cpb[idx++] = cpu_to_le16(IGN);
fbbb262d
RH		 811
812 return idx;
813}
814
5bd28a4b 815static int nv_adma_check_cpb(struct ata_port *ap, int cpb_num, int force_err)
fbbb262d
RH		 816{
817 struct nv_adma_port_priv *pp = ap->private_data;
2dec7555 818 u8 flags = pp->cpb[cpb_num].resp_flags;
fbbb262d
RH		 819
820 VPRINTK("CPB %d, flags=0x%x\n", cpb_num, flags);
821
5bd28a4b
RH		 822 if (unlikely((force_err ||
823 flags & (NV_CPB_RESP_ATA_ERR |
824 NV_CPB_RESP_CMD_ERR |
825 NV_CPB_RESP_CPB_ERR)))) {
9af5c9c9 826 struct ata_eh_info *ehi = &ap->link.eh_info;
5bd28a4b
RH		 827 int freeze = 0;
828
829 ata_ehi_clear_desc(ehi);
2dcb407e 830 __ata_ehi_push_desc(ehi, "CPB resp_flags 0x%x: ", flags);
5bd28a4b 831 if (flags & NV_CPB_RESP_ATA_ERR) {
b64bbc39 832 ata_ehi_push_desc(ehi, "ATA error");
5bd28a4b
RH		 833 ehi->err_mask |= AC_ERR_DEV;
834 } else if (flags & NV_CPB_RESP_CMD_ERR) {
b64bbc39 835 ata_ehi_push_desc(ehi, "CMD error");
5bd28a4b
RH		 836 ehi->err_mask |= AC_ERR_DEV;
837 } else if (flags & NV_CPB_RESP_CPB_ERR) {
b64bbc39 838 ata_ehi_push_desc(ehi, "CPB error");
5bd28a4b
RH		 839 ehi->err_mask |= AC_ERR_SYSTEM;
840 freeze = 1;
841 } else {
842 /* notifier error, but no error in CPB flags? */
b64bbc39 843 ata_ehi_push_desc(ehi, "unknown");
5bd28a4b
RH		 844 ehi->err_mask |= AC_ERR_OTHER;
845 freeze = 1;
846 }
847 /* Kill all commands. EH will determine what actually failed. */
848 if (freeze)
849 ata_port_freeze(ap);
850 else
851 ata_port_abort(ap);
852 return 1;
fbbb262d 853 }
5bd28a4b 854
f2fb344b 855 if (likely(flags & NV_CPB_RESP_DONE)) {
fbbb262d 856 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, cpb_num);
5bd28a4b
RH		 857 VPRINTK("CPB flags done, flags=0x%x\n", flags);
858 if (likely(qc)) {
2dcb407e 859 DPRINTK("Completing qc from tag %d\n", cpb_num);
fbbb262d 860 ata_qc_complete(qc);
2a54cf76 861 } else {
9af5c9c9 862 struct ata_eh_info *ehi = &ap->link.eh_info;
2a54cf76
RH		 863 /* Notifier bits set without a command may indicate the drive
864 is misbehaving. Raise host state machine violation on this
865 condition. */
5796d1c4
JG		 866 ata_port_printk(ap, KERN_ERR,
867 "notifier for tag %d with no cmd?\n",
868 cpb_num);
2a54cf76 869 ehi->err_mask |= AC_ERR_HSM;
cf480626 870 ehi->action |= ATA_EH_RESET;
2a54cf76
RH		 871 ata_port_freeze(ap);
872 return 1;
fbbb262d
RH		 873 }
874 }
5bd28a4b 875 return 0;
fbbb262d
RH		 876}
877
2dec7555
RH		 878static int nv_host_intr(struct ata_port *ap, u8 irq_stat)
879{
9af5c9c9 880 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->link.active_tag);
2dec7555
RH		 881
882 /* freeze if hotplugged */
883 if (unlikely(irq_stat & (NV_INT_ADDED | NV_INT_REMOVED))) {
884 ata_port_freeze(ap);
885 return 1;
886 }
887
888 /* bail out if not our interrupt */
889 if (!(irq_stat & NV_INT_DEV))
890 return 0;
891
892 /* DEV interrupt w/ no active qc? */
893 if (unlikely(!qc || (qc->tf.flags & ATA_TFLAG_POLLING))) {
9363c382 894 ata_sff_check_status(ap);
2dec7555
RH		 895 return 1;
896 }
897
898 /* handle interrupt */
9363c382 899 return ata_sff_host_intr(ap, qc);
2dec7555
RH		 900}
901
fbbb262d
RH		 902static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance)
903{
904 struct ata_host *host = dev_instance;
905 int i, handled = 0;
2dec7555 906 u32 notifier_clears[2];
fbbb262d
RH		 907
908 spin_lock(&host->lock);
909
910 for (i = 0; i < host->n_ports; i++) {
911 struct ata_port *ap = host->ports[i];
2dec7555 912 notifier_clears[i] = 0;
fbbb262d
RH		 913
914 if (ap && !(ap->flags & ATA_FLAG_DISABLED)) {
915 struct nv_adma_port_priv *pp = ap->private_data;
cdf56bcf 916 void __iomem *mmio = pp->ctl_block;
fbbb262d
RH		 917 u16 status;
918 u32 gen_ctl;
fbbb262d 919 u32 notifier, notifier_error;
a617c09f 920
53014e25
RH		 921 /* if ADMA is disabled, use standard ata interrupt handler */
922 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
923 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
924 >> (NV_INT_PORT_SHIFT * i);
925 handled += nv_host_intr(ap, irq_stat);
926 continue;
927 }
fbbb262d 928
53014e25 929 /* if in ATA register mode, check for standard interrupts */
fbbb262d 930 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) {
0d5ff566 931 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
2dec7555 932 >> (NV_INT_PORT_SHIFT * i);
2dcb407e 933 if (ata_tag_valid(ap->link.active_tag))
f740d168
RH		 934 /** NV_INT_DEV indication seems unreliable at times
935 at least in ADMA mode. Force it on always when a
936 command is active, to prevent losing interrupts. */
937 irq_stat |= NV_INT_DEV;
2dec7555 938 handled += nv_host_intr(ap, irq_stat);
fbbb262d
RH		 939 }
940
941 notifier = readl(mmio + NV_ADMA_NOTIFIER);
942 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
2dec7555 943 notifier_clears[i] = notifier | notifier_error;
fbbb262d 944
cdf56bcf 945 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
fbbb262d 946
2dcb407e 947 if (!NV_ADMA_CHECK_INTR(gen_ctl, ap->port_no) && !notifier &&
fbbb262d
RH		 948 !notifier_error)
949 /* Nothing to do */
950 continue;
951
952 status = readw(mmio + NV_ADMA_STAT);
953
954 /* Clear status. Ensure the controller sees the clearing before we start
955 looking at any of the CPB statuses, so that any CPB completions after
956 this point in the handler will raise another interrupt. */
957 writew(status, mmio + NV_ADMA_STAT);
958 readw(mmio + NV_ADMA_STAT); /* flush posted write */
959 rmb();
960
5bd28a4b
RH		 961 handled++; /* irq handled if we got here */
962
963 /* freeze if hotplugged or controller error */
964 if (unlikely(status & (NV_ADMA_STAT_HOTPLUG |
965 NV_ADMA_STAT_HOTUNPLUG |
5278b50c
RH		 966 NV_ADMA_STAT_TIMEOUT |
967 NV_ADMA_STAT_SERROR))) {
9af5c9c9 968 struct ata_eh_info *ehi = &ap->link.eh_info;
5bd28a4b
RH		 969
970 ata_ehi_clear_desc(ehi);
2dcb407e 971 __ata_ehi_push_desc(ehi, "ADMA status 0x%08x: ", status);
5bd28a4b
RH		 972 if (status & NV_ADMA_STAT_TIMEOUT) {
973 ehi->err_mask |= AC_ERR_SYSTEM;
b64bbc39 974 ata_ehi_push_desc(ehi, "timeout");
5bd28a4b
RH		 975 } else if (status & NV_ADMA_STAT_HOTPLUG) {
976 ata_ehi_hotplugged(ehi);
b64bbc39 977 ata_ehi_push_desc(ehi, "hotplug");
5bd28a4b
RH		 978 } else if (status & NV_ADMA_STAT_HOTUNPLUG) {
979 ata_ehi_hotplugged(ehi);
b64bbc39 980 ata_ehi_push_desc(ehi, "hot unplug");
5278b50c
RH		 981 } else if (status & NV_ADMA_STAT_SERROR) {
982 /* let libata analyze SError and figure out the cause */
b64bbc39
TH		 983 ata_ehi_push_desc(ehi, "SError");
984 } else
985 ata_ehi_push_desc(ehi, "unknown");
fbbb262d 986 ata_port_freeze(ap);
fbbb262d
RH		 987 continue;
988 }
989
5bd28a4b 990 if (status & (NV_ADMA_STAT_DONE |
a1fe7824
RH		 991 NV_ADMA_STAT_CPBERR |
992 NV_ADMA_STAT_CMD_COMPLETE)) {
993 u32 check_commands = notifier_clears[i];
721449bf 994 int pos, error = 0;
8ba5e4cb 995
a1fe7824
RH		 996 if (status & NV_ADMA_STAT_CPBERR) {
997 /* Check all active commands */
998 if (ata_tag_valid(ap->link.active_tag))
999 check_commands = 1 <<
1000 ap->link.active_tag;
1001 else
1002 check_commands = ap->
1003 link.sactive;
1004 }
8ba5e4cb 1005
fbbb262d 1006 /** Check CPBs for completed commands */
721449bf
RH		 1007 while ((pos = ffs(check_commands)) && !error) {
1008 pos--;
1009 error = nv_adma_check_cpb(ap, pos,
5796d1c4
JG		 1010 notifier_error & (1 << pos));
1011 check_commands &= ~(1 << pos);
fbbb262d
RH		 1012 }
1013 }
fbbb262d
RH		 1014 }
1015 }
f20b16ff 1016
b447916e 1017 if (notifier_clears[0] || notifier_clears[1]) {
2dec7555
RH		 1018 /* Note: Both notifier clear registers must be written
1019 if either is set, even if one is zero, according to NVIDIA. */
cdf56bcf
RH		 1020 struct nv_adma_port_priv *pp = host->ports[0]->private_data;
1021 writel(notifier_clears[0], pp->notifier_clear_block);
1022 pp = host->ports[1]->private_data;
1023 writel(notifier_clears[1], pp->notifier_clear_block);
2dec7555 1024 }
fbbb262d
RH		 1025
1026 spin_unlock(&host->lock);
1027
1028 return IRQ_RETVAL(handled);
1029}
1030
53014e25
RH		 1031static void nv_adma_freeze(struct ata_port *ap)
1032{
1033 struct nv_adma_port_priv *pp = ap->private_data;
1034 void __iomem *mmio = pp->ctl_block;
1035 u16 tmp;
1036
1037 nv_ck804_freeze(ap);
1038
1039 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1040 return;
1041
1042 /* clear any outstanding CK804 notifications */
2dcb407e 1043 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
53014e25
RH		 1044 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1045
1046 /* Disable interrupt */
1047 tmp = readw(mmio + NV_ADMA_CTL);
2dcb407e 1048 writew(tmp & ~(NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
53014e25 1049 mmio + NV_ADMA_CTL);
5796d1c4 1050 readw(mmio + NV_ADMA_CTL); /* flush posted write */
53014e25
RH		 1051}
1052
1053static void nv_adma_thaw(struct ata_port *ap)
1054{
1055 struct nv_adma_port_priv *pp = ap->private_data;
1056 void __iomem *mmio = pp->ctl_block;
1057 u16 tmp;
1058
1059 nv_ck804_thaw(ap);
1060
1061 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1062 return;
1063
1064 /* Enable interrupt */
1065 tmp = readw(mmio + NV_ADMA_CTL);
2dcb407e 1066 writew(tmp | (NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
53014e25 1067 mmio + NV_ADMA_CTL);
5796d1c4 1068 readw(mmio + NV_ADMA_CTL); /* flush posted write */
53014e25
RH		 1069}
1070
fbbb262d
RH		 1071static void nv_adma_irq_clear(struct ata_port *ap)
1072{
cdf56bcf
RH		 1073 struct nv_adma_port_priv *pp = ap->private_data;
1074 void __iomem *mmio = pp->ctl_block;
53014e25 1075 u32 notifier_clears[2];
fbbb262d 1076
53014e25 1077 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
9363c382 1078 ata_sff_irq_clear(ap);
53014e25
RH		 1079 return;
1080 }
1081
1082 /* clear any outstanding CK804 notifications */
2dcb407e 1083 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
53014e25 1084 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
fbbb262d 1085
53014e25
RH		 1086 /* clear ADMA status */
1087 writew(0xffff, mmio + NV_ADMA_STAT);
a617c09f 1088
53014e25
RH		 1089 /* clear notifiers - note both ports need to be written with
1090 something even though we are only clearing on one */
1091 if (ap->port_no == 0) {
1092 notifier_clears[0] = 0xFFFFFFFF;
1093 notifier_clears[1] = 0;
1094 } else {
1095 notifier_clears[0] = 0;
1096 notifier_clears[1] = 0xFFFFFFFF;
1097 }
1098 pp = ap->host->ports[0]->private_data;
1099 writel(notifier_clears[0], pp->notifier_clear_block);
1100 pp = ap->host->ports[1]->private_data;
1101 writel(notifier_clears[1], pp->notifier_clear_block);
fbbb262d
RH		 1102}
1103
f5ecac2d 1104static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc)
fbbb262d 1105{
f5ecac2d 1106 struct nv_adma_port_priv *pp = qc->ap->private_data;
fbbb262d 1107
b447916e 1108 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
9363c382 1109 ata_sff_post_internal_cmd(qc);
fbbb262d
RH		 1110}
1111
1112static int nv_adma_port_start(struct ata_port *ap)
1113{
1114 struct device *dev = ap->host->dev;
1115 struct nv_adma_port_priv *pp;
1116 int rc;
1117 void *mem;
1118 dma_addr_t mem_dma;
cdf56bcf 1119 void __iomem *mmio;
8959d300 1120 struct pci_dev *pdev = to_pci_dev(dev);
fbbb262d
RH		 1121 u16 tmp;
1122
1123 VPRINTK("ENTER\n");
1124
8959d300
RH		 1125 /* Ensure DMA mask is set to 32-bit before allocating legacy PRD and
1126 pad buffers */
1127 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1128 if (rc)
1129 return rc;
1130 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1131 if (rc)
1132 return rc;
1133
fbbb262d
RH		 1134 rc = ata_port_start(ap);
1135 if (rc)
1136 return rc;
1137
24dc5f33
TH		 1138 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1139 if (!pp)
1140 return -ENOMEM;
fbbb262d 1141
0d5ff566 1142 mmio = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_PORT +
cdf56bcf
RH		 1143 ap->port_no * NV_ADMA_PORT_SIZE;
1144 pp->ctl_block = mmio;
0d5ff566 1145 pp->gen_block = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_GEN;
cdf56bcf
RH		 1146 pp->notifier_clear_block = pp->gen_block +
1147 NV_ADMA_NOTIFIER_CLEAR + (4 * ap->port_no);
1148
8959d300
RH		 1149 /* Now that the legacy PRD and padding buffer are allocated we can
1150 safely raise the DMA mask to allocate the CPB/APRD table.
1151 These are allowed to fail since we store the value that ends up
1152 being used to set as the bounce limit in slave_config later if
1153 needed. */
1154 pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1155 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1156 pp->adma_dma_mask = *dev->dma_mask;
1157
24dc5f33
TH		 1158 mem = dmam_alloc_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ,
1159 &mem_dma, GFP_KERNEL);
1160 if (!mem)
1161 return -ENOMEM;
fbbb262d
RH		 1162 memset(mem, 0, NV_ADMA_PORT_PRIV_DMA_SZ);
1163
1164 /*
1165 * First item in chunk of DMA memory:
1166 * 128-byte command parameter block (CPB)
1167 * one for each command tag
1168 */
1169 pp->cpb = mem;
1170 pp->cpb_dma = mem_dma;
1171
1172 writel(mem_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
5796d1c4 1173 writel((mem_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
fbbb262d
RH		 1174
1175 mem += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1176 mem_dma += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1177
1178 /*
1179 * Second item: block of ADMA_SGTBL_LEN s/g entries
1180 */
1181 pp->aprd = mem;
1182 pp->aprd_dma = mem_dma;
1183
1184 ap->private_data = pp;
1185
1186 /* clear any outstanding interrupt conditions */
1187 writew(0xffff, mmio + NV_ADMA_STAT);
1188
1189 /* initialize port variables */
1190 pp->flags = NV_ADMA_PORT_REGISTER_MODE;
1191
1192 /* clear CPB fetch count */
1193 writew(0, mmio + NV_ADMA_CPB_COUNT);
1194
cdf56bcf 1195 /* clear GO for register mode, enable interrupt */
fbbb262d 1196 tmp = readw(mmio + NV_ADMA_CTL);
5796d1c4
JG		 1197 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1198 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
fbbb262d
RH		 1199
1200 tmp = readw(mmio + NV_ADMA_CTL);
1201 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
5796d1c4 1202 readw(mmio + NV_ADMA_CTL); /* flush posted write */
fbbb262d
RH		 1203 udelay(1);
1204 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
5796d1c4 1205 readw(mmio + NV_ADMA_CTL); /* flush posted write */
fbbb262d
RH		 1206
1207 return 0;
fbbb262d
RH		 1208}
1209
1210static void nv_adma_port_stop(struct ata_port *ap)
1211{
fbbb262d 1212 struct nv_adma_port_priv *pp = ap->private_data;
cdf56bcf 1213 void __iomem *mmio = pp->ctl_block;
fbbb262d
RH		 1214
1215 VPRINTK("ENTER\n");
fbbb262d 1216 writew(0, mmio + NV_ADMA_CTL);
fbbb262d
RH		 1217}
1218
438ac6d5 1219#ifdef CONFIG_PM
cdf56bcf
RH		 1220static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg)
1221{
1222 struct nv_adma_port_priv *pp = ap->private_data;
1223 void __iomem *mmio = pp->ctl_block;
1224
1225 /* Go to register mode - clears GO */
1226 nv_adma_register_mode(ap);
1227
1228 /* clear CPB fetch count */
1229 writew(0, mmio + NV_ADMA_CPB_COUNT);
1230
1231 /* disable interrupt, shut down port */
1232 writew(0, mmio + NV_ADMA_CTL);
1233
1234 return 0;
1235}
1236
1237static int nv_adma_port_resume(struct ata_port *ap)
1238{
1239 struct nv_adma_port_priv *pp = ap->private_data;
1240 void __iomem *mmio = pp->ctl_block;
1241 u16 tmp;
1242
1243 /* set CPB block location */
1244 writel(pp->cpb_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
5796d1c4 1245 writel((pp->cpb_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
cdf56bcf
RH		 1246
1247 /* clear any outstanding interrupt conditions */
1248 writew(0xffff, mmio + NV_ADMA_STAT);
1249
1250 /* initialize port variables */
1251 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
1252
1253 /* clear CPB fetch count */
1254 writew(0, mmio + NV_ADMA_CPB_COUNT);
1255
1256 /* clear GO for register mode, enable interrupt */
1257 tmp = readw(mmio + NV_ADMA_CTL);
5796d1c4
JG		 1258 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1259 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
cdf56bcf
RH		 1260
1261 tmp = readw(mmio + NV_ADMA_CTL);
1262 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
5796d1c4 1263 readw(mmio + NV_ADMA_CTL); /* flush posted write */
cdf56bcf
RH		 1264 udelay(1);
1265 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
5796d1c4 1266 readw(mmio + NV_ADMA_CTL); /* flush posted write */
cdf56bcf
RH		 1267
1268 return 0;
1269}
438ac6d5 1270#endif
fbbb262d 1271
9a829ccf 1272static void nv_adma_setup_port(struct ata_port *ap)
fbbb262d 1273{
9a829ccf
TH		 1274 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1275 struct ata_ioports *ioport = &ap->ioaddr;
fbbb262d
RH		 1276
1277 VPRINTK("ENTER\n");
1278
9a829ccf 1279 mmio += NV_ADMA_PORT + ap->port_no * NV_ADMA_PORT_SIZE;
fbbb262d 1280
0d5ff566
TH		 1281 ioport->cmd_addr = mmio;
1282 ioport->data_addr = mmio + (ATA_REG_DATA * 4);
fbbb262d 1283 ioport->error_addr =
0d5ff566
TH		 1284 ioport->feature_addr = mmio + (ATA_REG_ERR * 4);
1285 ioport->nsect_addr = mmio + (ATA_REG_NSECT * 4);
1286 ioport->lbal_addr = mmio + (ATA_REG_LBAL * 4);
1287 ioport->lbam_addr = mmio + (ATA_REG_LBAM * 4);
1288 ioport->lbah_addr = mmio + (ATA_REG_LBAH * 4);
1289 ioport->device_addr = mmio + (ATA_REG_DEVICE * 4);
fbbb262d 1290 ioport->status_addr =
0d5ff566 1291 ioport->command_addr = mmio + (ATA_REG_STATUS * 4);
fbbb262d 1292 ioport->altstatus_addr =
0d5ff566 1293 ioport->ctl_addr = mmio + 0x20;
fbbb262d
RH		 1294}
1295
9a829ccf 1296static int nv_adma_host_init(struct ata_host *host)
fbbb262d 1297{
9a829ccf 1298 struct pci_dev *pdev = to_pci_dev(host->dev);
fbbb262d
RH		 1299 unsigned int i;
1300 u32 tmp32;
1301
1302 VPRINTK("ENTER\n");
1303
1304 /* enable ADMA on the ports */
1305 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1306 tmp32 |= NV_MCP_SATA_CFG_20_PORT0_EN |
1307 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
1308 NV_MCP_SATA_CFG_20_PORT1_EN |
1309 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
1310
1311 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1312
9a829ccf
TH		 1313 for (i = 0; i < host->n_ports; i++)
1314 nv_adma_setup_port(host->ports[i]);
fbbb262d 1315
fbbb262d
RH		 1316 return 0;
1317}
1318
1319static void nv_adma_fill_aprd(struct ata_queued_cmd *qc,
1320 struct scatterlist *sg,
1321 int idx,
1322 struct nv_adma_prd *aprd)
1323{
41949ed5 1324 u8 flags = 0;
fbbb262d
RH		 1325 if (qc->tf.flags & ATA_TFLAG_WRITE)
1326 flags |= NV_APRD_WRITE;
1327 if (idx == qc->n_elem - 1)
1328 flags |= NV_APRD_END;
1329 else if (idx != 4)
1330 flags |= NV_APRD_CONT;
1331
1332 aprd->addr = cpu_to_le64(((u64)sg_dma_address(sg)));
1333 aprd->len = cpu_to_le32(((u32)sg_dma_len(sg))); /* len in bytes */
2dec7555 1334 aprd->flags = flags;
41949ed5 1335 aprd->packet_len = 0;
fbbb262d
RH		 1336}
1337
1338static void nv_adma_fill_sg(struct ata_queued_cmd *qc, struct nv_adma_cpb *cpb)
1339{
1340 struct nv_adma_port_priv *pp = qc->ap->private_data;
fbbb262d
RH		 1341 struct nv_adma_prd *aprd;
1342 struct scatterlist *sg;
ff2aeb1e 1343 unsigned int si;
fbbb262d
RH		 1344
1345 VPRINTK("ENTER\n");
1346
ff2aeb1e
TH		 1347 for_each_sg(qc->sg, sg, qc->n_elem, si) {
1348 aprd = (si < 5) ? &cpb->aprd[si] :
1349 &pp->aprd[NV_ADMA_SGTBL_LEN * qc->tag + (si-5)];
1350 nv_adma_fill_aprd(qc, sg, si, aprd);
fbbb262d 1351 }
ff2aeb1e 1352 if (si > 5)
fbbb262d 1353 cpb->next_aprd = cpu_to_le64(((u64)(pp->aprd_dma + NV_ADMA_SGTBL_SZ * qc->tag)));
41949ed5
RH		 1354 else
1355 cpb->next_aprd = cpu_to_le64(0);
fbbb262d
RH		 1356}
1357
382a6652
RH		 1358static int nv_adma_use_reg_mode(struct ata_queued_cmd *qc)
1359{
1360 struct nv_adma_port_priv *pp = qc->ap->private_data;
1361
1362 /* ADMA engine can only be used for non-ATAPI DMA commands,
3f3debdb 1363 or interrupt-driven no-data commands. */
b447916e 1364 if ((pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
3f3debdb 1365 (qc->tf.flags & ATA_TFLAG_POLLING))
382a6652
RH		 1366 return 1;
1367
b447916e 1368 if ((qc->flags & ATA_QCFLAG_DMAMAP) ||
382a6652
RH		 1369 (qc->tf.protocol == ATA_PROT_NODATA))
1370 return 0;
1371
1372 return 1;
1373}
1374
fbbb262d
RH		 1375static void nv_adma_qc_prep(struct ata_queued_cmd *qc)
1376{
1377 struct nv_adma_port_priv *pp = qc->ap->private_data;
1378 struct nv_adma_cpb *cpb = &pp->cpb[qc->tag];
1379 u8 ctl_flags = NV_CPB_CTL_CPB_VALID |
fbbb262d
RH		 1380 NV_CPB_CTL_IEN;
1381
382a6652 1382 if (nv_adma_use_reg_mode(qc)) {
3f3debdb
RH		 1383 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1384 (qc->flags & ATA_QCFLAG_DMAMAP));
2dec7555 1385 nv_adma_register_mode(qc->ap);
9363c382 1386 ata_sff_qc_prep(qc);
fbbb262d
RH		 1387 return;
1388 }
1389
41949ed5
RH		 1390 cpb->resp_flags = NV_CPB_RESP_DONE;
1391 wmb();
1392 cpb->ctl_flags = 0;
1393 wmb();
fbbb262d
RH		 1394
1395 cpb->len = 3;
1396 cpb->tag = qc->tag;
1397 cpb->next_cpb_idx = 0;
1398
1399 /* turn on NCQ flags for NCQ commands */
1400 if (qc->tf.protocol == ATA_PROT_NCQ)
1401 ctl_flags |= NV_CPB_CTL_QUEUE | NV_CPB_CTL_FPDMA;
1402
cdf56bcf
RH		 1403 VPRINTK("qc->flags = 0x%lx\n", qc->flags);
1404
fbbb262d
RH		 1405 nv_adma_tf_to_cpb(&qc->tf, cpb->tf);
1406
b447916e 1407 if (qc->flags & ATA_QCFLAG_DMAMAP) {
382a6652
RH		 1408 nv_adma_fill_sg(qc, cpb);
1409 ctl_flags |= NV_CPB_CTL_APRD_VALID;
1410 } else
1411 memset(&cpb->aprd[0], 0, sizeof(struct nv_adma_prd) * 5);
fbbb262d 1412
5796d1c4
JG		 1413 /* Be paranoid and don't let the device see NV_CPB_CTL_CPB_VALID
1414 until we are finished filling in all of the contents */
fbbb262d
RH		 1415 wmb();
1416 cpb->ctl_flags = ctl_flags;
41949ed5
RH		 1417 wmb();
1418 cpb->resp_flags = 0;
fbbb262d
RH		 1419}
1420
1421static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc)
1422{
2dec7555 1423 struct nv_adma_port_priv *pp = qc->ap->private_data;
cdf56bcf 1424 void __iomem *mmio = pp->ctl_block;
5e5c74a5 1425 int curr_ncq = (qc->tf.protocol == ATA_PROT_NCQ);
fbbb262d
RH		 1426
1427 VPRINTK("ENTER\n");
1428
3f3debdb
RH		 1429 /* We can't handle result taskfile with NCQ commands, since
1430 retrieving the taskfile switches us out of ADMA mode and would abort
1431 existing commands. */
1432 if (unlikely(qc->tf.protocol == ATA_PROT_NCQ &&
1433 (qc->flags & ATA_QCFLAG_RESULT_TF))) {
1434 ata_dev_printk(qc->dev, KERN_ERR,
1435 "NCQ w/ RESULT_TF not allowed\n");
1436 return AC_ERR_SYSTEM;
1437 }
1438
382a6652 1439 if (nv_adma_use_reg_mode(qc)) {
fbbb262d 1440 /* use ATA register mode */
382a6652 1441 VPRINTK("using ATA register mode: 0x%lx\n", qc->flags);
3f3debdb
RH		 1442 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1443 (qc->flags & ATA_QCFLAG_DMAMAP));
fbbb262d 1444 nv_adma_register_mode(qc->ap);
9363c382 1445 return ata_sff_qc_issue(qc);
fbbb262d
RH		 1446 } else
1447 nv_adma_mode(qc->ap);
1448
1449 /* write append register, command tag in lower 8 bits
1450 and (number of cpbs to append -1) in top 8 bits */
1451 wmb();
5e5c74a5 1452
b447916e 1453 if (curr_ncq != pp->last_issue_ncq) {
5796d1c4
JG		 1454 /* Seems to need some delay before switching between NCQ and
1455 non-NCQ commands, else we get command timeouts and such. */
5e5c74a5
RH		 1456 udelay(20);
1457 pp->last_issue_ncq = curr_ncq;
1458 }
1459
fbbb262d
RH		 1460 writew(qc->tag, mmio + NV_ADMA_APPEND);
1461
5796d1c4 1462 DPRINTK("Issued tag %u\n", qc->tag);
fbbb262d
RH		 1463
1464 return 0;
1465}
1466
7d12e780 1467static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance)
1da177e4 1468{
cca3974e 1469 struct ata_host *host = dev_instance;
1da177e4
LT		 1470 unsigned int i;
1471 unsigned int handled = 0;
1472 unsigned long flags;
1473
cca3974e 1474 spin_lock_irqsave(&host->lock, flags);
1da177e4 1475
cca3974e 1476 for (i = 0; i < host->n_ports; i++) {
1da177e4
LT		 1477 struct ata_port *ap;
1478
cca3974e 1479 ap = host->ports[i];
c1389503 1480 if (ap &&
029f5468 1481 !(ap->flags & ATA_FLAG_DISABLED)) {
1da177e4
LT		 1482 struct ata_queued_cmd *qc;
1483
9af5c9c9 1484 qc = ata_qc_from_tag(ap, ap->link.active_tag);
e50362ec 1485 if (qc && (!(qc->tf.flags & ATA_TFLAG_POLLING)))
9363c382 1486 handled += ata_sff_host_intr(ap, qc);
b887030a
AC		 1487 else
1488 // No request pending? Clear interrupt status
1489 // anyway, in case there's one pending.
5682ed33 1490 ap->ops->sff_check_status(ap);
1da177e4
LT		 1491 }
1492
1493 }
1494
cca3974e 1495 spin_unlock_irqrestore(&host->lock, flags);
1da177e4
LT		 1496
1497 return IRQ_RETVAL(handled);
1498}
1499
cca3974e 1500static irqreturn_t nv_do_interrupt(struct ata_host *host, u8 irq_stat)
ada364e8
TH		 1501{
1502 int i, handled = 0;
1503
cca3974e
JG		 1504 for (i = 0; i < host->n_ports; i++) {
1505 struct ata_port *ap = host->ports[i];
ada364e8
TH		 1506
1507 if (ap && !(ap->flags & ATA_FLAG_DISABLED))
1508 handled += nv_host_intr(ap, irq_stat);
1509
1510 irq_stat >>= NV_INT_PORT_SHIFT;
1511 }
1512
1513 return IRQ_RETVAL(handled);
1514}
1515
7d12e780 1516static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance)
ada364e8 1517{
cca3974e 1518 struct ata_host *host = dev_instance;
ada364e8
TH		 1519 u8 irq_stat;
1520 irqreturn_t ret;
1521
cca3974e 1522 spin_lock(&host->lock);
0d5ff566 1523 irq_stat = ioread8(host->ports[0]->ioaddr.scr_addr + NV_INT_STATUS);
cca3974e
JG		 1524 ret = nv_do_interrupt(host, irq_stat);
1525 spin_unlock(&host->lock);
ada364e8
TH		 1526
1527 return ret;
1528}
1529
7d12e780 1530static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance)
ada364e8 1531{
cca3974e 1532 struct ata_host *host = dev_instance;
ada364e8
TH		 1533 u8 irq_stat;
1534 irqreturn_t ret;
1535
cca3974e 1536 spin_lock(&host->lock);
0d5ff566 1537 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
cca3974e
JG		 1538 ret = nv_do_interrupt(host, irq_stat);
1539 spin_unlock(&host->lock);
ada364e8
TH		 1540
1541 return ret;
1542}
1543
82ef04fb 1544static int nv_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val)
1da177e4 1545{
1da177e4 1546 if (sc_reg > SCR_CONTROL)
da3dbb17 1547 return -EINVAL;
1da177e4 1548
82ef04fb 1549 *val = ioread32(link->ap->ioaddr.scr_addr + (sc_reg * 4));
da3dbb17 1550 return 0;
1da177e4
LT		 1551}
1552
82ef04fb 1553static int nv_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val)
1da177e4 1554{
1da177e4 1555 if (sc_reg > SCR_CONTROL)
da3dbb17 1556 return -EINVAL;
1da177e4 1557
82ef04fb 1558 iowrite32(val, link->ap->ioaddr.scr_addr + (sc_reg * 4));
da3dbb17 1559 return 0;
1da177e4
LT		 1560}
1561
e8caa3c7
TH		 1562static int nv_noclassify_hardreset(struct ata_link *link, unsigned int *class,
1563 unsigned long deadline)
1564{
1565 bool online;
1566 int rc;
1567
1568 rc = sata_link_hardreset(link, sata_deb_timing_hotplug, deadline,
1569 &online, NULL);
1570 return online ? -EAGAIN : rc;
1571}
1572
39f87582
TH		 1573static void nv_nf2_freeze(struct ata_port *ap)
1574{
0d5ff566 1575 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
39f87582
TH		 1576 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1577 u8 mask;
1578
0d5ff566 1579 mask = ioread8(scr_addr + NV_INT_ENABLE);
39f87582 1580 mask &= ~(NV_INT_ALL << shift);
0d5ff566 1581 iowrite8(mask, scr_addr + NV_INT_ENABLE);
39f87582
TH		 1582}
1583
1584static void nv_nf2_thaw(struct ata_port *ap)
1585{
0d5ff566 1586 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
39f87582
TH		 1587 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1588 u8 mask;
1589
0d5ff566 1590 iowrite8(NV_INT_ALL << shift, scr_addr + NV_INT_STATUS);
39f87582 1591
0d5ff566 1592 mask = ioread8(scr_addr + NV_INT_ENABLE);
39f87582 1593 mask |= (NV_INT_MASK << shift);
0d5ff566 1594 iowrite8(mask, scr_addr + NV_INT_ENABLE);
39f87582
TH		 1595}
1596
1597static void nv_ck804_freeze(struct ata_port *ap)
1598{
0d5ff566 1599 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
39f87582
TH		 1600 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1601 u8 mask;
1602
1603 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1604 mask &= ~(NV_INT_ALL << shift);
1605 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1606}
1607
1608static void nv_ck804_thaw(struct ata_port *ap)
1609{
0d5ff566 1610 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
39f87582
TH		 1611 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1612 u8 mask;
1613
1614 writeb(NV_INT_ALL << shift, mmio_base + NV_INT_STATUS_CK804);
1615
1616 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1617 mask |= (NV_INT_MASK << shift);
1618 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1619}
1620
f140f0f1
KL		 1621static void nv_mcp55_freeze(struct ata_port *ap)
1622{
1623 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1624 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1625 u32 mask;
1626
1627 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1628
1629 mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1630 mask &= ~(NV_INT_ALL_MCP55 << shift);
1631 writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
9363c382 1632 ata_sff_freeze(ap);
f140f0f1
KL		 1633}
1634
1635static void nv_mcp55_thaw(struct ata_port *ap)
1636{
1637 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1638 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1639 u32 mask;
1640
1641 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1642
1643 mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1644 mask |= (NV_INT_MASK_MCP55 << shift);
1645 writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
9363c382 1646 ata_sff_thaw(ap);
f140f0f1
KL		 1647}
1648
fbbb262d
RH		 1649static void nv_adma_error_handler(struct ata_port *ap)
1650{
1651 struct nv_adma_port_priv *pp = ap->private_data;
b447916e 1652 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {
cdf56bcf 1653 void __iomem *mmio = pp->ctl_block;
fbbb262d
RH		 1654 int i;
1655 u16 tmp;
a84471fe 1656
b447916e 1657 if (ata_tag_valid(ap->link.active_tag) || ap->link.sactive) {
2cb27853
RH		 1658 u32 notifier = readl(mmio + NV_ADMA_NOTIFIER);
1659 u32 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
1660 u32 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
1661 u32 status = readw(mmio + NV_ADMA_STAT);
08af7414
RH		 1662 u8 cpb_count = readb(mmio + NV_ADMA_CPB_COUNT);
1663 u8 next_cpb_idx = readb(mmio + NV_ADMA_NEXT_CPB_IDX);
2cb27853 1664
5796d1c4
JG		 1665 ata_port_printk(ap, KERN_ERR,
1666 "EH in ADMA mode, notifier 0x%X "
08af7414
RH		 1667 "notifier_error 0x%X gen_ctl 0x%X status 0x%X "
1668 "next cpb count 0x%X next cpb idx 0x%x\n",
1669 notifier, notifier_error, gen_ctl, status,
1670 cpb_count, next_cpb_idx);
2cb27853 1671
b447916e 1672 for (i = 0; i < NV_ADMA_MAX_CPBS; i++) {
2cb27853 1673 struct nv_adma_cpb *cpb = &pp->cpb[i];
b447916e 1674 if ((ata_tag_valid(ap->link.active_tag) && i == ap->link.active_tag) ||
5796d1c4 1675 ap->link.sactive & (1 << i))
2cb27853
RH		 1676 ata_port_printk(ap, KERN_ERR,
1677 "CPB %d: ctl_flags 0x%x, resp_flags 0x%x\n",
1678 i, cpb->ctl_flags, cpb->resp_flags);
1679 }
1680 }
fbbb262d 1681
fbbb262d
RH		 1682 /* Push us back into port register mode for error handling. */
1683 nv_adma_register_mode(ap);
1684
5796d1c4
JG		 1685 /* Mark all of the CPBs as invalid to prevent them from
1686 being executed */
b447916e 1687 for (i = 0; i < NV_ADMA_MAX_CPBS; i++)
fbbb262d
RH		 1688 pp->cpb[i].ctl_flags &= ~NV_CPB_CTL_CPB_VALID;
1689
1690 /* clear CPB fetch count */
1691 writew(0, mmio + NV_ADMA_CPB_COUNT);
1692
1693 /* Reset channel */
1694 tmp = readw(mmio + NV_ADMA_CTL);
1695 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
b447916e 1696 readw(mmio + NV_ADMA_CTL); /* flush posted write */
fbbb262d
RH		 1697 udelay(1);
1698 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
b447916e 1699 readw(mmio + NV_ADMA_CTL); /* flush posted write */
fbbb262d
RH		 1700 }
1701
9363c382 1702 ata_sff_error_handler(ap);
fbbb262d
RH		 1703}
1704
f140f0f1
KL		 1705static void nv_swncq_qc_to_dq(struct ata_port *ap, struct ata_queued_cmd *qc)
1706{
1707 struct nv_swncq_port_priv *pp = ap->private_data;
1708 struct defer_queue *dq = &pp->defer_queue;
1709
1710 /* queue is full */
1711 WARN_ON(dq->tail - dq->head == ATA_MAX_QUEUE);
1712 dq->defer_bits |= (1 << qc->tag);
1713 dq->tag[dq->tail++ & (ATA_MAX_QUEUE - 1)] = qc->tag;
1714}
1715
1716static struct ata_queued_cmd *nv_swncq_qc_from_dq(struct ata_port *ap)
1717{
1718 struct nv_swncq_port_priv *pp = ap->private_data;
1719 struct defer_queue *dq = &pp->defer_queue;
1720 unsigned int tag;
1721
1722 if (dq->head == dq->tail) /* null queue */
1723 return NULL;
1724
1725 tag = dq->tag[dq->head & (ATA_MAX_QUEUE - 1)];
1726 dq->tag[dq->head++ & (ATA_MAX_QUEUE - 1)] = ATA_TAG_POISON;
1727 WARN_ON(!(dq->defer_bits & (1 << tag)));
1728 dq->defer_bits &= ~(1 << tag);
1729
1730 return ata_qc_from_tag(ap, tag);
1731}
1732
1733static void nv_swncq_fis_reinit(struct ata_port *ap)
1734{
1735 struct nv_swncq_port_priv *pp = ap->private_data;
1736
1737 pp->dhfis_bits = 0;
1738 pp->dmafis_bits = 0;
1739 pp->sdbfis_bits = 0;
1740 pp->ncq_flags = 0;
1741}
1742
1743static void nv_swncq_pp_reinit(struct ata_port *ap)
1744{
1745 struct nv_swncq_port_priv *pp = ap->private_data;
1746 struct defer_queue *dq = &pp->defer_queue;
1747
1748 dq->head = 0;
1749 dq->tail = 0;
1750 dq->defer_bits = 0;
1751 pp->qc_active = 0;
1752 pp->last_issue_tag = ATA_TAG_POISON;
1753 nv_swncq_fis_reinit(ap);
1754}
1755
1756static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis)
1757{
1758 struct nv_swncq_port_priv *pp = ap->private_data;
1759
1760 writew(fis, pp->irq_block);
1761}
1762
1763static void __ata_bmdma_stop(struct ata_port *ap)
1764{
1765 struct ata_queued_cmd qc;
1766
1767 qc.ap = ap;
1768 ata_bmdma_stop(&qc);
1769}
1770
1771static void nv_swncq_ncq_stop(struct ata_port *ap)
1772{
1773 struct nv_swncq_port_priv *pp = ap->private_data;
1774 unsigned int i;
1775 u32 sactive;
1776 u32 done_mask;
1777
1778 ata_port_printk(ap, KERN_ERR,
1779 "EH in SWNCQ mode,QC:qc_active 0x%X sactive 0x%X\n",
1780 ap->qc_active, ap->link.sactive);
1781 ata_port_printk(ap, KERN_ERR,
1782 "SWNCQ:qc_active 0x%X defer_bits 0x%X last_issue_tag 0x%x\n "
1783 "dhfis 0x%X dmafis 0x%X sdbfis 0x%X\n",
1784 pp->qc_active, pp->defer_queue.defer_bits, pp->last_issue_tag,
1785 pp->dhfis_bits, pp->dmafis_bits, pp->sdbfis_bits);
1786
1787 ata_port_printk(ap, KERN_ERR, "ATA_REG 0x%X ERR_REG 0x%X\n",
5682ed33 1788 ap->ops->sff_check_status(ap),
f140f0f1
KL		 1789 ioread8(ap->ioaddr.error_addr));
1790
1791 sactive = readl(pp->sactive_block);
1792 done_mask = pp->qc_active ^ sactive;
1793
1794 ata_port_printk(ap, KERN_ERR, "tag : dhfis dmafis sdbfis sacitve\n");
1795 for (i = 0; i < ATA_MAX_QUEUE; i++) {
1796 u8 err = 0;
1797 if (pp->qc_active & (1 << i))
1798 err = 0;
1799 else if (done_mask & (1 << i))
1800 err = 1;
1801 else
1802 continue;
1803
1804 ata_port_printk(ap, KERN_ERR,
1805 "tag 0x%x: %01x %01x %01x %01x %s\n", i,
1806 (pp->dhfis_bits >> i) & 0x1,
1807 (pp->dmafis_bits >> i) & 0x1,
1808 (pp->sdbfis_bits >> i) & 0x1,
1809 (sactive >> i) & 0x1,
1810 (err ? "error! tag doesn't exit" : " "));
1811 }
1812
1813 nv_swncq_pp_reinit(ap);
5682ed33 1814 ap->ops->sff_irq_clear(ap);
f140f0f1
KL		 1815 __ata_bmdma_stop(ap);
1816 nv_swncq_irq_clear(ap, 0xffff);
1817}
1818
1819static void nv_swncq_error_handler(struct ata_port *ap)
1820{
1821 struct ata_eh_context *ehc = &ap->link.eh_context;
1822
1823 if (ap->link.sactive) {
1824 nv_swncq_ncq_stop(ap);
cf480626 1825 ehc->i.action |= ATA_EH_RESET;
f140f0f1
KL		 1826 }
1827
9363c382 1828 ata_sff_error_handler(ap);
f140f0f1
KL		 1829}
1830
1831#ifdef CONFIG_PM
1832static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg)
1833{
1834 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1835 u32 tmp;
1836
1837 /* clear irq */
1838 writel(~0, mmio + NV_INT_STATUS_MCP55);
1839
1840 /* disable irq */
1841 writel(0, mmio + NV_INT_ENABLE_MCP55);
1842
1843 /* disable swncq */
1844 tmp = readl(mmio + NV_CTL_MCP55);
1845 tmp &= ~(NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ);
1846 writel(tmp, mmio + NV_CTL_MCP55);
1847
1848 return 0;
1849}
1850
1851static int nv_swncq_port_resume(struct ata_port *ap)
1852{
1853 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1854 u32 tmp;
1855
1856 /* clear irq */
1857 writel(~0, mmio + NV_INT_STATUS_MCP55);
1858
1859 /* enable irq */
1860 writel(0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1861
1862 /* enable swncq */
1863 tmp = readl(mmio + NV_CTL_MCP55);
1864 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1865
1866 return 0;
1867}
1868#endif
1869
1870static void nv_swncq_host_init(struct ata_host *host)
1871{
1872 u32 tmp;
1873 void __iomem *mmio = host->iomap[NV_MMIO_BAR];
1874 struct pci_dev *pdev = to_pci_dev(host->dev);
1875 u8 regval;
1876
1877 /* disable ECO 398 */
1878 pci_read_config_byte(pdev, 0x7f, &regval);
1879 regval &= ~(1 << 7);
1880 pci_write_config_byte(pdev, 0x7f, regval);
1881
1882 /* enable swncq */
1883 tmp = readl(mmio + NV_CTL_MCP55);
1884 VPRINTK("HOST_CTL:0x%X\n", tmp);
1885 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1886
1887 /* enable irq intr */
1888 tmp = readl(mmio + NV_INT_ENABLE_MCP55);
1889 VPRINTK("HOST_ENABLE:0x%X\n", tmp);
1890 writel(tmp | 0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1891
1892 /* clear port irq */
1893 writel(~0x0, mmio + NV_INT_STATUS_MCP55);
1894}
1895
1896static int nv_swncq_slave_config(struct scsi_device *sdev)
1897{
1898 struct ata_port *ap = ata_shost_to_port(sdev->host);
1899 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
1900 struct ata_device *dev;
1901 int rc;
1902 u8 rev;
1903 u8 check_maxtor = 0;
1904 unsigned char model_num[ATA_ID_PROD_LEN + 1];
1905
1906 rc = ata_scsi_slave_config(sdev);
1907 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
1908 /* Not a proper libata device, ignore */
1909 return rc;
1910
1911 dev = &ap->link.device[sdev->id];
1912 if (!(ap->flags & ATA_FLAG_NCQ) || dev->class == ATA_DEV_ATAPI)
1913 return rc;
1914
1915 /* if MCP51 and Maxtor, then disable ncq */
1916 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA ||
1917 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2)
1918 check_maxtor = 1;
1919
1920 /* if MCP55 and rev <= a2 and Maxtor, then disable ncq */
1921 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA ||
1922 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2) {
1923 pci_read_config_byte(pdev, 0x8, &rev);
1924 if (rev <= 0xa2)
1925 check_maxtor = 1;
1926 }
1927
1928 if (!check_maxtor)
1929 return rc;
1930
1931 ata_id_c_string(dev->id, model_num, ATA_ID_PROD, sizeof(model_num));
1932
1933 if (strncmp(model_num, "Maxtor", 6) == 0) {
1934 ata_scsi_change_queue_depth(sdev, 1);
1935 ata_dev_printk(dev, KERN_NOTICE,
1936 "Disabling SWNCQ mode (depth %x)\n", sdev->queue_depth);
1937 }
1938
1939 return rc;
1940}
1941
1942static int nv_swncq_port_start(struct ata_port *ap)
1943{
1944 struct device *dev = ap->host->dev;
1945 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1946 struct nv_swncq_port_priv *pp;
1947 int rc;
1948
1949 rc = ata_port_start(ap);
1950 if (rc)
1951 return rc;
1952
1953 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1954 if (!pp)
1955 return -ENOMEM;
1956
1957 pp->prd = dmam_alloc_coherent(dev, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE,
1958 &pp->prd_dma, GFP_KERNEL);
1959 if (!pp->prd)
1960 return -ENOMEM;
1961 memset(pp->prd, 0, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE);
1962
1963 ap->private_data = pp;
1964 pp->sactive_block = ap->ioaddr.scr_addr + 4 * SCR_ACTIVE;
1965 pp->irq_block = mmio + NV_INT_STATUS_MCP55 + ap->port_no * 2;
1966 pp->tag_block = mmio + NV_NCQ_REG_MCP55 + ap->port_no * 2;
1967
1968 return 0;
1969}
1970
1971static void nv_swncq_qc_prep(struct ata_queued_cmd *qc)
1972{
1973 if (qc->tf.protocol != ATA_PROT_NCQ) {
9363c382 1974 ata_sff_qc_prep(qc);
f140f0f1
KL		 1975 return;
1976 }
1977
1978 if (!(qc->flags & ATA_QCFLAG_DMAMAP))
1979 return;
1980
1981 nv_swncq_fill_sg(qc);
1982}
1983
1984static void nv_swncq_fill_sg(struct ata_queued_cmd *qc)
1985{
1986 struct ata_port *ap = qc->ap;
1987 struct scatterlist *sg;
f140f0f1
KL		 1988 struct nv_swncq_port_priv *pp = ap->private_data;
1989 struct ata_prd *prd;
ff2aeb1e 1990 unsigned int si, idx;
f140f0f1
KL		 1991
1992 prd = pp->prd + ATA_MAX_PRD * qc->tag;
1993
1994 idx = 0;
ff2aeb1e 1995 for_each_sg(qc->sg, sg, qc->n_elem, si) {
f140f0f1
KL		 1996 u32 addr, offset;
1997 u32 sg_len, len;
1998
1999 addr = (u32)sg_dma_address(sg);
2000 sg_len = sg_dma_len(sg);
2001
2002 while (sg_len) {
2003 offset = addr & 0xffff;
2004 len = sg_len;
2005 if ((offset + sg_len) > 0x10000)
2006 len = 0x10000 - offset;
2007
2008 prd[idx].addr = cpu_to_le32(addr);
2009 prd[idx].flags_len = cpu_to_le32(len & 0xffff);
2010
2011 idx++;
2012 sg_len -= len;
2013 addr += len;
2014 }
2015 }
2016
ff2aeb1e 2017 prd[idx - 1].flags_len |= cpu_to_le32(ATA_PRD_EOT);
f140f0f1
KL		 2018}
2019
2020static unsigned int nv_swncq_issue_atacmd(struct ata_port *ap,
2021 struct ata_queued_cmd *qc)
2022{
2023 struct nv_swncq_port_priv *pp = ap->private_data;
2024
2025 if (qc == NULL)
2026 return 0;
2027
2028 DPRINTK("Enter\n");
2029
2030 writel((1 << qc->tag), pp->sactive_block);
2031 pp->last_issue_tag = qc->tag;
2032 pp->dhfis_bits &= ~(1 << qc->tag);
2033 pp->dmafis_bits &= ~(1 << qc->tag);
2034 pp->qc_active |= (0x1 << qc->tag);
2035
5682ed33
TH		 2036 ap->ops->sff_tf_load(ap, &qc->tf); /* load tf registers */
2037 ap->ops->sff_exec_command(ap, &qc->tf);
f140f0f1
KL		 2038
2039 DPRINTK("Issued tag %u\n", qc->tag);
2040
2041 return 0;
2042}
2043
2044static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc)
2045{
2046 struct ata_port *ap = qc->ap;
2047 struct nv_swncq_port_priv *pp = ap->private_data;
2048
2049 if (qc->tf.protocol != ATA_PROT_NCQ)
9363c382 2050 return ata_sff_qc_issue(qc);
f140f0f1
KL		 2051
2052 DPRINTK("Enter\n");
2053
2054 if (!pp->qc_active)
2055 nv_swncq_issue_atacmd(ap, qc);
2056 else
2057 nv_swncq_qc_to_dq(ap, qc); /* add qc to defer queue */
2058
2059 return 0;
2060}
2061
2062static void nv_swncq_hotplug(struct ata_port *ap, u32 fis)
2063{
2064 u32 serror;
2065 struct ata_eh_info *ehi = &ap->link.eh_info;
2066
2067 ata_ehi_clear_desc(ehi);
2068
2069 /* AHCI needs SError cleared; otherwise, it might lock up */
2070 sata_scr_read(&ap->link, SCR_ERROR, &serror);
2071 sata_scr_write(&ap->link, SCR_ERROR, serror);
2072
2073 /* analyze @irq_stat */
2074 if (fis & NV_SWNCQ_IRQ_ADDED)
2075 ata_ehi_push_desc(ehi, "hot plug");
2076 else if (fis & NV_SWNCQ_IRQ_REMOVED)
2077 ata_ehi_push_desc(ehi, "hot unplug");
2078
2079 ata_ehi_hotplugged(ehi);
2080
2081 /* okay, let's hand over to EH */
2082 ehi->serror |= serror;
2083
2084 ata_port_freeze(ap);
2085}
2086
2087static int nv_swncq_sdbfis(struct ata_port *ap)
2088{
2089 struct ata_queued_cmd *qc;
2090 struct nv_swncq_port_priv *pp = ap->private_data;
2091 struct ata_eh_info *ehi = &ap->link.eh_info;
2092 u32 sactive;
2093 int nr_done = 0;
2094 u32 done_mask;
2095 int i;
2096 u8 host_stat;
2097 u8 lack_dhfis = 0;
2098
2099 host_stat = ap->ops->bmdma_status(ap);
2100 if (unlikely(host_stat & ATA_DMA_ERR)) {
2101 /* error when transfering data to/from memory */
2102 ata_ehi_clear_desc(ehi);
2103 ata_ehi_push_desc(ehi, "BMDMA stat 0x%x", host_stat);
2104 ehi->err_mask |= AC_ERR_HOST_BUS;
cf480626 2105 ehi->action |= ATA_EH_RESET;
f140f0f1
KL		 2106 return -EINVAL;
2107 }
2108
5682ed33 2109 ap->ops->sff_irq_clear(ap);
f140f0f1
KL		 2110 __ata_bmdma_stop(ap);
2111
2112 sactive = readl(pp->sactive_block);
2113 done_mask = pp->qc_active ^ sactive;
2114
2115 if (unlikely(done_mask & sactive)) {
2116 ata_ehi_clear_desc(ehi);
2117 ata_ehi_push_desc(ehi, "illegal SWNCQ:qc_active transition"
2118 "(%08x->%08x)", pp->qc_active, sactive);
2119 ehi->err_mask |= AC_ERR_HSM;
cf480626 2120 ehi->action |= ATA_EH_RESET;
f140f0f1
KL		 2121 return -EINVAL;
2122 }
2123 for (i = 0; i < ATA_MAX_QUEUE; i++) {
2124 if (!(done_mask & (1 << i)))
2125 continue;
2126
2127 qc = ata_qc_from_tag(ap, i);
2128 if (qc) {
2129 ata_qc_complete(qc);
2130 pp->qc_active &= ~(1 << i);
2131 pp->dhfis_bits &= ~(1 << i);
2132 pp->dmafis_bits &= ~(1 << i);
2133 pp->sdbfis_bits |= (1 << i);
2134 nr_done++;
2135 }
2136 }
2137
2138 if (!ap->qc_active) {
2139 DPRINTK("over\n");
2140 nv_swncq_pp_reinit(ap);
2141 return nr_done;
2142 }
2143
2144 if (pp->qc_active & pp->dhfis_bits)
2145 return nr_done;
2146
2147 if ((pp->ncq_flags & ncq_saw_backout) ||
2148 (pp->qc_active ^ pp->dhfis_bits))
2149 /* if the controller cann't get a device to host register FIS,
2150 * The driver needs to reissue the new command.
2151 */
2152 lack_dhfis = 1;
2153
2154 DPRINTK("id 0x%x QC: qc_active 0x%x,"
2155 "SWNCQ:qc_active 0x%X defer_bits %X "
2156 "dhfis 0x%X dmafis 0x%X last_issue_tag %x\n",
2157 ap->print_id, ap->qc_active, pp->qc_active,
2158 pp->defer_queue.defer_bits, pp->dhfis_bits,
2159 pp->dmafis_bits, pp->last_issue_tag);
2160
2161 nv_swncq_fis_reinit(ap);
2162
2163 if (lack_dhfis) {
2164 qc = ata_qc_from_tag(ap, pp->last_issue_tag);
2165 nv_swncq_issue_atacmd(ap, qc);
2166 return nr_done;
2167 }
2168
2169 if (pp->defer_queue.defer_bits) {
2170 /* send deferral queue command */
2171 qc = nv_swncq_qc_from_dq(ap);
2172 WARN_ON(qc == NULL);
2173 nv_swncq_issue_atacmd(ap, qc);
2174 }
2175
2176 return nr_done;
2177}
2178
2179static inline u32 nv_swncq_tag(struct ata_port *ap)
2180{
2181 struct nv_swncq_port_priv *pp = ap->private_data;
2182 u32 tag;
2183
2184 tag = readb(pp->tag_block) >> 2;
2185 return (tag & 0x1f);
2186}
2187
2188static int nv_swncq_dmafis(struct ata_port *ap)
2189{
2190 struct ata_queued_cmd *qc;
2191 unsigned int rw;
2192 u8 dmactl;
2193 u32 tag;
2194 struct nv_swncq_port_priv *pp = ap->private_data;
2195
2196 __ata_bmdma_stop(ap);
2197 tag = nv_swncq_tag(ap);
2198
2199 DPRINTK("dma setup tag 0x%x\n", tag);
2200 qc = ata_qc_from_tag(ap, tag);
2201
2202 if (unlikely(!qc))
2203 return 0;
2204
2205 rw = qc->tf.flags & ATA_TFLAG_WRITE;
2206
2207 /* load PRD table addr. */
2208 iowrite32(pp->prd_dma + ATA_PRD_TBL_SZ * qc->tag,
2209 ap->ioaddr.bmdma_addr + ATA_DMA_TABLE_OFS);
2210
2211 /* specify data direction, triple-check start bit is clear */
2212 dmactl = ioread8(ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2213 dmactl &= ~ATA_DMA_WR;
2214 if (!rw)
2215 dmactl |= ATA_DMA_WR;
2216
2217 iowrite8(dmactl | ATA_DMA_START, ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2218
2219 return 1;
2220}
2221
2222static void nv_swncq_host_interrupt(struct ata_port *ap, u16 fis)
2223{
2224 struct nv_swncq_port_priv *pp = ap->private_data;
2225 struct ata_queued_cmd *qc;
2226 struct ata_eh_info *ehi = &ap->link.eh_info;
2227 u32 serror;
2228 u8 ata_stat;
2229 int rc = 0;
2230
5682ed33 2231 ata_stat = ap->ops->sff_check_status(ap);
f140f0f1
KL		 2232 nv_swncq_irq_clear(ap, fis);
2233 if (!fis)
2234 return;
2235
2236 if (ap->pflags & ATA_PFLAG_FROZEN)
2237 return;
2238
2239 if (fis & NV_SWNCQ_IRQ_HOTPLUG) {
2240 nv_swncq_hotplug(ap, fis);
2241 return;
2242 }
2243
2244 if (!pp->qc_active)
2245 return;
2246
82ef04fb 2247 if (ap->ops->scr_read(&ap->link, SCR_ERROR, &serror))
f140f0f1 2248 return;
82ef04fb 2249 ap->ops->scr_write(&ap->link, SCR_ERROR, serror);
f140f0f1
KL		 2250
2251 if (ata_stat & ATA_ERR) {
2252 ata_ehi_clear_desc(ehi);
2253 ata_ehi_push_desc(ehi, "Ata error. fis:0x%X", fis);
2254 ehi->err_mask |= AC_ERR_DEV;
2255 ehi->serror |= serror;
cf480626 2256 ehi->action |= ATA_EH_RESET;
f140f0f1
KL		 2257 ata_port_freeze(ap);
2258 return;
2259 }
2260
2261 if (fis & NV_SWNCQ_IRQ_BACKOUT) {
2262 /* If the IRQ is backout, driver must issue
2263 * the new command again some time later.
2264 */
2265 pp->ncq_flags |= ncq_saw_backout;
2266 }
2267
2268 if (fis & NV_SWNCQ_IRQ_SDBFIS) {
2269 pp->ncq_flags |= ncq_saw_sdb;
2270 DPRINTK("id 0x%x SWNCQ: qc_active 0x%X "
2271 "dhfis 0x%X dmafis 0x%X sactive 0x%X\n",
2272 ap->print_id, pp->qc_active, pp->dhfis_bits,
2273 pp->dmafis_bits, readl(pp->sactive_block));
2274 rc = nv_swncq_sdbfis(ap);
2275 if (rc < 0)
2276 goto irq_error;
2277 }
2278
2279 if (fis & NV_SWNCQ_IRQ_DHREGFIS) {
2280 /* The interrupt indicates the new command
2281 * was transmitted correctly to the drive.
2282 */
2283 pp->dhfis_bits |= (0x1 << pp->last_issue_tag);
2284 pp->ncq_flags |= ncq_saw_d2h;
2285 if (pp->ncq_flags & (ncq_saw_sdb | ncq_saw_backout)) {
2286 ata_ehi_push_desc(ehi, "illegal fis transaction");
2287 ehi->err_mask |= AC_ERR_HSM;
cf480626 2288 ehi->action |= ATA_EH_RESET;
f140f0f1
KL		 2289 goto irq_error;
2290 }
2291
2292 if (!(fis & NV_SWNCQ_IRQ_DMASETUP) &&
2293 !(pp->ncq_flags & ncq_saw_dmas)) {
5682ed33 2294 ata_stat = ap->ops->sff_check_status(ap);
f140f0f1
KL		 2295 if (ata_stat & ATA_BUSY)
2296 goto irq_exit;
2297
2298 if (pp->defer_queue.defer_bits) {
2299 DPRINTK("send next command\n");
2300 qc = nv_swncq_qc_from_dq(ap);
2301 nv_swncq_issue_atacmd(ap, qc);
2302 }
2303 }
2304 }
2305
2306 if (fis & NV_SWNCQ_IRQ_DMASETUP) {
2307 /* program the dma controller with appropriate PRD buffers
2308 * and start the DMA transfer for requested command.
2309 */
2310 pp->dmafis_bits |= (0x1 << nv_swncq_tag(ap));
2311 pp->ncq_flags |= ncq_saw_dmas;
2312 rc = nv_swncq_dmafis(ap);
2313 }
2314
2315irq_exit:
2316 return;
2317irq_error:
2318 ata_ehi_push_desc(ehi, "fis:0x%x", fis);
2319 ata_port_freeze(ap);
2320 return;
2321}
2322
2323static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance)
2324{
2325 struct ata_host *host = dev_instance;
2326 unsigned int i;
2327 unsigned int handled = 0;
2328 unsigned long flags;
2329 u32 irq_stat;
2330
2331 spin_lock_irqsave(&host->lock, flags);
2332
2333 irq_stat = readl(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_MCP55);
2334
2335 for (i = 0; i < host->n_ports; i++) {
2336 struct ata_port *ap = host->ports[i];
2337
2338 if (ap && !(ap->flags & ATA_FLAG_DISABLED)) {
2339 if (ap->link.sactive) {
2340 nv_swncq_host_interrupt(ap, (u16)irq_stat);
2341 handled = 1;
2342 } else {
2343 if (irq_stat) /* reserve Hotplug */
2344 nv_swncq_irq_clear(ap, 0xfff0);
2345
2346 handled += nv_host_intr(ap, (u8)irq_stat);
2347 }
2348 }
2349 irq_stat >>= NV_INT_PORT_SHIFT_MCP55;
2350 }
2351
2352 spin_unlock_irqrestore(&host->lock, flags);
2353
2354 return IRQ_RETVAL(handled);
2355}
2356
5796d1c4 2357static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
1da177e4 2358{
5796d1c4 2359 static int printed_version;
1626aeb8 2360 const struct ata_port_info *ppi[] = { NULL, NULL };
95947193 2361 struct nv_pi_priv *ipriv;
9a829ccf 2362 struct ata_host *host;
cdf56bcf 2363 struct nv_host_priv *hpriv;
1da177e4
LT		 2364 int rc;
2365 u32 bar;
0d5ff566 2366 void __iomem *base;
fbbb262d 2367 unsigned long type = ent->driver_data;
1da177e4
LT		 2368
2369 // Make sure this is a SATA controller by counting the number of bars
2370 // (NVIDIA SATA controllers will always have six bars). Otherwise,
2371 // it's an IDE controller and we ignore it.
5796d1c4 2372 for (bar = 0; bar < 6; bar++)
1da177e4
LT		 2373 if (pci_resource_start(pdev, bar) == 0)
2374 return -ENODEV;
2375
cdf56bcf 2376 if (!printed_version++)
a9524a76 2377 dev_printk(KERN_DEBUG, &pdev->dev, "version " DRV_VERSION "\n");
1da177e4 2378
24dc5f33 2379 rc = pcim_enable_device(pdev);
1da177e4 2380 if (rc)
24dc5f33 2381 return rc;
1da177e4 2382
9a829ccf 2383 /* determine type and allocate host */
f140f0f1 2384 if (type == CK804 && adma_enabled) {
fbbb262d
RH		 2385 dev_printk(KERN_NOTICE, &pdev->dev, "Using ADMA mode\n");
2386 type = ADMA;
2d775708
TH		 2387 } else if (type == MCP5x && swncq_enabled) {
2388 dev_printk(KERN_NOTICE, &pdev->dev, "Using SWNCQ mode\n");
2389 type = SWNCQ;
360737a9
JG		 2390 }
2391
1626aeb8 2392 ppi[0] = &nv_port_info[type];
95947193 2393 ipriv = ppi[0]->private_data;
9363c382 2394 rc = ata_pci_sff_prepare_host(pdev, ppi, &host);
9a829ccf
TH		 2395 if (rc)
2396 return rc;
1da177e4 2397
24dc5f33 2398 hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL);
cdf56bcf 2399 if (!hpriv)
24dc5f33 2400 return -ENOMEM;
9a829ccf
TH		 2401 hpriv->type = type;
2402 host->private_data = hpriv;
cdf56bcf 2403
9a829ccf
TH		 2404 /* request and iomap NV_MMIO_BAR */
2405 rc = pcim_iomap_regions(pdev, 1 << NV_MMIO_BAR, DRV_NAME);
2406 if (rc)
2407 return rc;
1da177e4 2408
9a829ccf
TH		 2409 /* configure SCR access */
2410 base = host->iomap[NV_MMIO_BAR];
2411 host->ports[0]->ioaddr.scr_addr = base + NV_PORT0_SCR_REG_OFFSET;
2412 host->ports[1]->ioaddr.scr_addr = base + NV_PORT1_SCR_REG_OFFSET;
1da177e4 2413
ada364e8 2414 /* enable SATA space for CK804 */
fbbb262d 2415 if (type >= CK804) {
ada364e8
TH		 2416 u8 regval;
2417
2418 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2419 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2420 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2421 }
2422
9a829ccf 2423 /* init ADMA */
fbbb262d 2424 if (type == ADMA) {
9a829ccf 2425 rc = nv_adma_host_init(host);
fbbb262d 2426 if (rc)
24dc5f33 2427 return rc;
360737a9 2428 } else if (type == SWNCQ)
f140f0f1 2429 nv_swncq_host_init(host);
fbbb262d 2430
9a829ccf 2431 pci_set_master(pdev);
95947193
TH		 2432 return ata_host_activate(host, pdev->irq, ipriv->irq_handler,
2433 IRQF_SHARED, ipriv->sht);
1da177e4
LT		 2434}
2435
438ac6d5 2436#ifdef CONFIG_PM
cdf56bcf
RH		 2437static int nv_pci_device_resume(struct pci_dev *pdev)
2438{
2439 struct ata_host *host = dev_get_drvdata(&pdev->dev);
2440 struct nv_host_priv *hpriv = host->private_data;
ce053fa8 2441 int rc;
cdf56bcf 2442
ce053fa8 2443 rc = ata_pci_device_do_resume(pdev);
b447916e 2444 if (rc)
ce053fa8 2445 return rc;
cdf56bcf
RH		 2446
2447 if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
b447916e 2448 if (hpriv->type >= CK804) {
cdf56bcf
RH		 2449 u8 regval;
2450
2451 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2452 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2453 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2454 }
b447916e 2455 if (hpriv->type == ADMA) {
cdf56bcf
RH		 2456 u32 tmp32;
2457 struct nv_adma_port_priv *pp;
2458 /* enable/disable ADMA on the ports appropriately */
2459 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2460
2461 pp = host->ports[0]->private_data;
b447916e 2462 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
cdf56bcf 2463 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
5796d1c4 2464 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
cdf56bcf
RH		 2465 else
2466 tmp32 |= (NV_MCP_SATA_CFG_20_PORT0_EN |
5796d1c4 2467 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
cdf56bcf 2468 pp = host->ports[1]->private_data;
b447916e 2469 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
cdf56bcf 2470 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT1_EN |
5796d1c4 2471 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
cdf56bcf
RH		 2472 else
2473 tmp32 |= (NV_MCP_SATA_CFG_20_PORT1_EN |
5796d1c4 2474 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
cdf56bcf
RH		 2475
2476 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2477 }
2478 }
2479
2480 ata_host_resume(host);
2481
2482 return 0;
2483}
438ac6d5 2484#endif
cdf56bcf 2485
cca3974e 2486static void nv_ck804_host_stop(struct ata_host *host)
ada364e8 2487{
cca3974e 2488 struct pci_dev *pdev = to_pci_dev(host->dev);
ada364e8
TH		 2489 u8 regval;
2490
2491 /* disable SATA space for CK804 */
2492 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2493 regval &= ~NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2494 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
ada364e8
TH		 2495}
2496
fbbb262d
RH		 2497static void nv_adma_host_stop(struct ata_host *host)
2498{
2499 struct pci_dev *pdev = to_pci_dev(host->dev);
fbbb262d
RH		 2500 u32 tmp32;
2501
fbbb262d
RH		 2502 /* disable ADMA on the ports */
2503 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2504 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
2505 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
2506 NV_MCP_SATA_CFG_20_PORT1_EN |
2507 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2508
2509 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2510
2511 nv_ck804_host_stop(host);
2512}
2513
1da177e4
LT		 2514static int __init nv_init(void)
2515{
b7887196 2516 return pci_register_driver(&nv_pci_driver);
1da177e4
LT		 2517}
2518
2519static void __exit nv_exit(void)
2520{
2521 pci_unregister_driver(&nv_pci_driver);
2522}
2523
2524module_init(nv_init);
2525module_exit(nv_exit);
fbbb262d 2526module_param_named(adma, adma_enabled, bool, 0444);
55f784c8 2527MODULE_PARM_DESC(adma, "Enable use of ADMA (Default: false)");
f140f0f1 2528module_param_named(swncq, swncq_enabled, bool, 0444);
d21279f4 2529MODULE_PARM_DESC(swncq, "Enable use of SWNCQ (Default: true)");
f140f0f1 2530
Clone of davem's net-next-2.6 tree