blob: 261a7d24b6a45ecb50c721a8490540d8c44e2698 [file] [log] [blame]
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +01001// Low level AHCI disk access
2//
3// Copyright (C) 2010 Gerd Hoffmann <kraxel@redhat.com>
4//
5// This file may be distributed under the terms of the GNU LGPLv3 license.
6
Kevin O'Connor2d2fa312013-09-14 21:55:26 -04007#include "ahci.h" // CDB_CMD_READ_10
8#include "ata.h" // ATA_CB_STAT
Kevin O'Connor4bc49972012-05-13 22:58:08 -04009#include "biosvar.h" // GET_GLOBAL
Kevin O'Connor2d2fa312013-09-14 21:55:26 -040010#include "blockcmd.h" // CDB_CMD_READ_10
Kevin O'Connor2d2fa312013-09-14 21:55:26 -040011#include "malloc.h" // free
12#include "output.h" // dprintf
Kevin O'Connor4d8510c2016-02-03 01:28:20 -050013#include "pci.h" // pci_config_readb
14#include "pcidevice.h" // foreachpci
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +010015#include "pci_ids.h" // PCI_CLASS_STORAGE_OTHER
16#include "pci_regs.h" // PCI_INTERRUPT_LINE
Kevin O'Connor3df600b2013-09-14 19:28:55 -040017#include "stacks.h" // yield
Kevin O'Connor135f3f62013-09-14 23:57:26 -040018#include "std/disk.h" // DISK_RET_SUCCESS
Kevin O'Connorfa9c66a2013-09-14 19:10:40 -040019#include "string.h" // memset
Kevin O'Connor2d2fa312013-09-14 21:55:26 -040020#include "util.h" // timer_calc
Kevin O'Connor4ade5232013-09-18 21:41:48 -040021#include "x86.h" // inb
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +010022
Gerd Hoffmanne1041192011-07-14 16:24:04 +020023#define AHCI_REQUEST_TIMEOUT 32000 // 32 seconds max for IDE ops
24#define AHCI_RESET_TIMEOUT 500 // 500 miliseconds
25#define AHCI_LINK_TIMEOUT 10 // 10 miliseconds
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +010026
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +010027// prepare sata command fis
28static void sata_prep_simple(struct sata_cmd_fis *fis, u8 command)
29{
30 memset_fl(fis, 0, sizeof(*fis));
Gerd Hoffmann26ae90b2013-11-26 14:01:20 +010031 fis->command = command;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +010032}
33
34static void sata_prep_readwrite(struct sata_cmd_fis *fis,
35 struct disk_op_s *op, int iswrite)
36{
37 u64 lba = op->lba;
38 u8 command;
39
40 memset_fl(fis, 0, sizeof(*fis));
41
42 if (op->count >= (1<<8) || lba + op->count >= (1<<28)) {
Gerd Hoffmann26ae90b2013-11-26 14:01:20 +010043 fis->sector_count2 = op->count >> 8;
44 fis->lba_low2 = lba >> 24;
45 fis->lba_mid2 = lba >> 32;
46 fis->lba_high2 = lba >> 40;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +010047 lba &= 0xffffff;
48 command = (iswrite ? ATA_CMD_WRITE_DMA_EXT
49 : ATA_CMD_READ_DMA_EXT);
50 } else {
51 command = (iswrite ? ATA_CMD_WRITE_DMA
52 : ATA_CMD_READ_DMA);
53 }
Gerd Hoffmann26ae90b2013-11-26 14:01:20 +010054 fis->feature = 1; /* dma */
55 fis->command = command;
56 fis->sector_count = op->count;
57 fis->lba_low = lba;
58 fis->lba_mid = lba >> 8;
59 fis->lba_high = lba >> 16;
60 fis->device = ((lba >> 24) & 0xf) | ATA_CB_DH_LBA;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +010061}
62
63static void sata_prep_atapi(struct sata_cmd_fis *fis, u16 blocksize)
64{
65 memset_fl(fis, 0, sizeof(*fis));
Gerd Hoffmann26ae90b2013-11-26 14:01:20 +010066 fis->command = ATA_CMD_PACKET;
67 fis->feature = 1; /* dma */
68 fis->lba_mid = blocksize;
69 fis->lba_high = blocksize >> 8;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +010070}
71
72// ahci register access helpers
73static u32 ahci_ctrl_readl(struct ahci_ctrl_s *ctrl, u32 reg)
74{
Kevin O'Connor6e3436b2016-02-02 22:11:30 -050075 return readl(ctrl->iobase + reg);
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +010076}
77
78static void ahci_ctrl_writel(struct ahci_ctrl_s *ctrl, u32 reg, u32 val)
79{
Kevin O'Connor6e3436b2016-02-02 22:11:30 -050080 writel(ctrl->iobase + reg, val);
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +010081}
82
83static u32 ahci_port_to_ctrl(u32 pnr, u32 port_reg)
84{
85 u32 ctrl_reg = 0x100;
86 ctrl_reg += pnr * 0x80;
87 ctrl_reg += port_reg;
88 return ctrl_reg;
89}
90
91static u32 ahci_port_readl(struct ahci_ctrl_s *ctrl, u32 pnr, u32 reg)
92{
93 u32 ctrl_reg = ahci_port_to_ctrl(pnr, reg);
94 return ahci_ctrl_readl(ctrl, ctrl_reg);
95}
96
97static void ahci_port_writel(struct ahci_ctrl_s *ctrl, u32 pnr, u32 reg, u32 val)
98{
99 u32 ctrl_reg = ahci_port_to_ctrl(pnr, reg);
100 ahci_ctrl_writel(ctrl, ctrl_reg, val);
101}
102
103// submit ahci command + wait for result
Kevin O'Connor1902c942013-10-26 11:48:06 -0400104static int ahci_command(struct ahci_port_s *port_gf, int iswrite, int isatapi,
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100105 void *buffer, u32 bsize)
106{
Gerd Hoffmann6f850492011-07-14 16:24:01 +0200107 u32 val, status, success, flags, intbits, error;
Gerd Hoffmann26ae90b2013-11-26 14:01:20 +0100108 struct ahci_ctrl_s *ctrl = port_gf->ctrl;
109 struct ahci_cmd_s *cmd = port_gf->cmd;
110 struct ahci_fis_s *fis = port_gf->fis;
111 struct ahci_list_s *list = port_gf->list;
112 u32 pnr = port_gf->pnr;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100113
Gerd Hoffmann26ae90b2013-11-26 14:01:20 +0100114 cmd->fis.reg = 0x27;
Gerd Hoffmann4a0f3662013-11-26 14:02:54 +0100115 cmd->fis.pmp_type = 1 << 7; /* cmd fis */
116 cmd->prdt[0].base = (u32)buffer;
Gerd Hoffmann26ae90b2013-11-26 14:01:20 +0100117 cmd->prdt[0].baseu = 0;
118 cmd->prdt[0].flags = bsize-1;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100119
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100120 flags = ((1 << 16) | /* one prd entry */
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100121 (iswrite ? (1 << 6) : 0) |
122 (isatapi ? (1 << 5) : 0) |
Gerd Hoffmanna8c6a4e2011-07-14 16:23:59 +0200123 (5 << 0)); /* fis length (dwords) */
Gerd Hoffmann26ae90b2013-11-26 14:01:20 +0100124 list[0].flags = flags;
125 list[0].bytes = 0;
Gerd Hoffmann4a0f3662013-11-26 14:02:54 +0100126 list[0].base = (u32)(cmd);
Gerd Hoffmann26ae90b2013-11-26 14:01:20 +0100127 list[0].baseu = 0;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100128
Kevin O'Connorbf079e12012-03-11 11:19:52 -0400129 dprintf(8, "AHCI/%d: send cmd ...\n", pnr);
Gerd Hoffmann07532972011-07-14 16:24:00 +0200130 intbits = ahci_port_readl(ctrl, pnr, PORT_IRQ_STAT);
131 if (intbits)
132 ahci_port_writel(ctrl, pnr, PORT_IRQ_STAT, intbits);
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100133 ahci_port_writel(ctrl, pnr, PORT_SCR_ACT, 1);
134 ahci_port_writel(ctrl, pnr, PORT_CMD_ISSUE, 1);
Gerd Hoffmann07532972011-07-14 16:24:00 +0200135
Kevin O'Connor018bdd72013-07-20 18:22:57 -0400136 u32 end = timer_calc(AHCI_REQUEST_TIMEOUT);
Gerd Hoffmann07532972011-07-14 16:24:00 +0200137 do {
138 for (;;) {
139 intbits = ahci_port_readl(ctrl, pnr, PORT_IRQ_STAT);
140 if (intbits) {
141 ahci_port_writel(ctrl, pnr, PORT_IRQ_STAT, intbits);
142 if (intbits & 0x02) {
Kevin O'Connor890c0852012-05-24 23:55:00 -0400143 status = GET_LOWFLAT(fis->psfis[2]);
144 error = GET_LOWFLAT(fis->psfis[3]);
Gerd Hoffmann07532972011-07-14 16:24:00 +0200145 break;
146 }
147 if (intbits & 0x01) {
Kevin O'Connor890c0852012-05-24 23:55:00 -0400148 status = GET_LOWFLAT(fis->rfis[2]);
149 error = GET_LOWFLAT(fis->rfis[3]);
Gerd Hoffmann07532972011-07-14 16:24:00 +0200150 break;
151 }
152 }
Kevin O'Connor018bdd72013-07-20 18:22:57 -0400153 if (timer_check(end)) {
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200154 warn_timeout();
155 return -1;
156 }
Gerd Hoffmann07532972011-07-14 16:24:00 +0200157 yield();
158 }
Kevin O'Connorbf079e12012-03-11 11:19:52 -0400159 dprintf(8, "AHCI/%d: ... intbits 0x%x, status 0x%x ...\n",
Gerd Hoffmann07532972011-07-14 16:24:00 +0200160 pnr, intbits, status);
161 } while (status & ATA_CB_STAT_BSY);
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100162
163 success = (0x00 == (status & (ATA_CB_STAT_BSY | ATA_CB_STAT_DF |
Gerd Hoffmanncbda7952011-07-14 16:24:03 +0200164 ATA_CB_STAT_ERR)) &&
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100165 ATA_CB_STAT_RDY == (status & (ATA_CB_STAT_RDY)));
Gerd Hoffmann6f850492011-07-14 16:24:01 +0200166 if (success) {
Kevin O'Connorbf079e12012-03-11 11:19:52 -0400167 dprintf(8, "AHCI/%d: ... finished, status 0x%x, OK\n", pnr,
Gerd Hoffmann6f850492011-07-14 16:24:01 +0200168 status);
169 } else {
170 dprintf(2, "AHCI/%d: ... finished, status 0x%x, ERROR 0x%x\n", pnr,
171 status, error);
172
173 // non-queued error recovery (AHCI 1.3 section 6.2.2.1)
174 // Clears PxCMD.ST to 0 to reset the PxCI register
175 val = ahci_port_readl(ctrl, pnr, PORT_CMD);
176 ahci_port_writel(ctrl, pnr, PORT_CMD, val & ~PORT_CMD_START);
177
178 // waits for PxCMD.CR to clear to 0
179 while (1) {
180 val = ahci_port_readl(ctrl, pnr, PORT_CMD);
181 if ((val & PORT_CMD_LIST_ON) == 0)
182 break;
183 yield();
184 }
185
186 // Clears any error bits in PxSERR to enable capturing new errors
187 val = ahci_port_readl(ctrl, pnr, PORT_SCR_ERR);
188 ahci_port_writel(ctrl, pnr, PORT_SCR_ERR, val);
189
190 // Clears status bits in PxIS as appropriate
191 val = ahci_port_readl(ctrl, pnr, PORT_IRQ_STAT);
192 ahci_port_writel(ctrl, pnr, PORT_IRQ_STAT, val);
193
194 // If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to 1, issue
195 // a COMRESET to the device to put it in an idle state
196 val = ahci_port_readl(ctrl, pnr, PORT_TFDATA);
197 if (val & (ATA_CB_STAT_BSY | ATA_CB_STAT_DRQ)) {
198 dprintf(2, "AHCI/%d: issue comreset\n", pnr);
199 val = ahci_port_readl(ctrl, pnr, PORT_SCR_CTL);
200 // set Device Detection Initialization (DET) to 1 for 1 ms for comreset
201 ahci_port_writel(ctrl, pnr, PORT_SCR_CTL, val | 1);
202 mdelay (1);
203 ahci_port_writel(ctrl, pnr, PORT_SCR_CTL, val);
204 }
205
206 // Sets PxCMD.ST to 1 to enable issuing new commands
207 val = ahci_port_readl(ctrl, pnr, PORT_CMD);
208 ahci_port_writel(ctrl, pnr, PORT_CMD, val | PORT_CMD_START);
209 }
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100210 return success ? 0 : -1;
211}
212
213#define CDROM_CDB_SIZE 12
214
Kevin O'Connor0e5c7702015-07-07 11:24:27 -0400215int ahci_atapi_process_op(struct disk_op_s *op)
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100216{
Kevin O'Connor80c2b6e2010-12-05 12:52:02 -0500217 if (! CONFIG_AHCI)
218 return 0;
219
Kevin O'Connor1902c942013-10-26 11:48:06 -0400220 struct ahci_port_s *port_gf = container_of(
221 op->drive_gf, struct ahci_port_s, drive);
Gerd Hoffmann26ae90b2013-11-26 14:01:20 +0100222 struct ahci_cmd_s *cmd = port_gf->cmd;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100223
Kevin O'Connor0e5c7702015-07-07 11:24:27 -0400224 if (op->command == CMD_WRITE || op->command == CMD_FORMAT)
225 return DISK_RET_EWRITEPROTECT;
226 int blocksize = scsi_fill_cmd(op, cmd->atapi, CDROM_CDB_SIZE);
227 if (blocksize < 0)
228 return default_process_op(op);
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100229 sata_prep_atapi(&cmd->fis, blocksize);
Kevin O'Connor0e5c7702015-07-07 11:24:27 -0400230 int rc = ahci_command(port_gf, 0, 1, op->buf_fl, op->count * blocksize);
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100231 if (rc < 0)
232 return DISK_RET_EBADTRACK;
233 return DISK_RET_SUCCESS;
234}
235
Scott Duplichan9c48aab2011-07-14 16:24:02 +0200236// read/write count blocks from a harddrive, op->buf_fl must be word aligned
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100237static int
Scott Duplichan9c48aab2011-07-14 16:24:02 +0200238ahci_disk_readwrite_aligned(struct disk_op_s *op, int iswrite)
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100239{
Kevin O'Connor1902c942013-10-26 11:48:06 -0400240 struct ahci_port_s *port_gf = container_of(
241 op->drive_gf, struct ahci_port_s, drive);
Gerd Hoffmann26ae90b2013-11-26 14:01:20 +0100242 struct ahci_cmd_s *cmd = port_gf->cmd;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100243 int rc;
244
245 sata_prep_readwrite(&cmd->fis, op, iswrite);
Kevin O'Connor1902c942013-10-26 11:48:06 -0400246 rc = ahci_command(port_gf, iswrite, 0, op->buf_fl,
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100247 op->count * DISK_SECTOR_SIZE);
Kevin O'Connorbf079e12012-03-11 11:19:52 -0400248 dprintf(8, "ahci disk %s, lba %6x, count %3x, buf %p, rc %d\n",
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100249 iswrite ? "write" : "read", (u32)op->lba, op->count, op->buf_fl, rc);
250 if (rc < 0)
251 return DISK_RET_EBADTRACK;
252 return DISK_RET_SUCCESS;
253}
254
Scott Duplichan9c48aab2011-07-14 16:24:02 +0200255// read/write count blocks from a harddrive.
256static int
257ahci_disk_readwrite(struct disk_op_s *op, int iswrite)
258{
259 // if caller's buffer is word aligned, use it directly
260 if (((u32) op->buf_fl & 1) == 0)
261 return ahci_disk_readwrite_aligned(op, iswrite);
262
263 // Use a word aligned buffer for AHCI I/O
264 int rc;
265 struct disk_op_s localop = *op;
Gerd Hoffmann26ae90b2013-11-26 14:01:20 +0100266 u8 *alignedbuf_fl = bounce_buf_fl;
Scott Duplichan9c48aab2011-07-14 16:24:02 +0200267 u8 *position = op->buf_fl;
268
269 localop.buf_fl = alignedbuf_fl;
270 localop.count = 1;
271
272 if (iswrite) {
273 u16 block;
274 for (block = 0; block < op->count; block++) {
275 memcpy_fl (alignedbuf_fl, position, DISK_SECTOR_SIZE);
276 rc = ahci_disk_readwrite_aligned (&localop, 1);
277 if (rc)
278 return rc;
279 position += DISK_SECTOR_SIZE;
280 localop.lba++;
281 }
282 } else { // read
283 u16 block;
284 for (block = 0; block < op->count; block++) {
285 rc = ahci_disk_readwrite_aligned (&localop, 0);
286 if (rc)
287 return rc;
288 memcpy_fl (position, alignedbuf_fl, DISK_SECTOR_SIZE);
289 position += DISK_SECTOR_SIZE;
290 localop.lba++;
291 }
292 }
293 return DISK_RET_SUCCESS;
294}
295
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100296// command demuxer
Kevin O'Connorc7fa7892015-07-07 08:35:51 -0400297int
Kevin O'Connor17856452015-07-07 14:56:20 -0400298ahci_process_op(struct disk_op_s *op)
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100299{
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100300 if (!CONFIG_AHCI)
301 return 0;
Kevin O'Connorbd6afe52012-07-21 12:01:12 -0400302 switch (op->command) {
303 case CMD_READ:
304 return ahci_disk_readwrite(op, 0);
305 case CMD_WRITE:
306 return ahci_disk_readwrite(op, 1);
Kevin O'Connorbd6afe52012-07-21 12:01:12 -0400307 default:
Kevin O'Connor85c72c62015-07-07 09:01:52 -0400308 return default_process_op(op);
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100309 }
310}
311
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100312static void
313ahci_port_reset(struct ahci_ctrl_s *ctrl, u32 pnr)
314{
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200315 u32 val;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100316
317 /* disable FIS + CMD */
Kevin O'Connor018bdd72013-07-20 18:22:57 -0400318 u32 end = timer_calc(AHCI_RESET_TIMEOUT);
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200319 for (;;) {
320 val = ahci_port_readl(ctrl, pnr, PORT_CMD);
321 if (!(val & (PORT_CMD_FIS_RX | PORT_CMD_START |
322 PORT_CMD_FIS_ON | PORT_CMD_LIST_ON)))
323 break;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100324 val &= ~(PORT_CMD_FIS_RX | PORT_CMD_START);
325 ahci_port_writel(ctrl, pnr, PORT_CMD, val);
Kevin O'Connor018bdd72013-07-20 18:22:57 -0400326 if (timer_check(end)) {
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200327 warn_timeout();
328 break;
329 }
330 yield();
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100331 }
332
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100333 /* disable + clear IRQs */
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200334 ahci_port_writel(ctrl, pnr, PORT_IRQ_MASK, 0);
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100335 val = ahci_port_readl(ctrl, pnr, PORT_IRQ_STAT);
336 if (val)
337 ahci_port_writel(ctrl, pnr, PORT_IRQ_STAT, val);
338}
339
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100340static struct ahci_port_s*
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200341ahci_port_alloc(struct ahci_ctrl_s *ctrl, u32 pnr)
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100342{
Gerd Hoffmannef8adc02011-08-04 19:36:31 +0200343 struct ahci_port_s *port = malloc_tmp(sizeof(*port));
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100344
345 if (!port) {
346 warn_noalloc();
347 return NULL;
348 }
349 port->pnr = pnr;
350 port->ctrl = ctrl;
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200351 port->list = memalign_tmp(1024, 1024);
352 port->fis = memalign_tmp(256, 256);
353 port->cmd = memalign_tmp(256, 256);
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100354 if (port->list == NULL || port->fis == NULL || port->cmd == NULL) {
355 warn_noalloc();
356 return NULL;
357 }
358 memset(port->list, 0, 1024);
359 memset(port->fis, 0, 256);
360 memset(port->cmd, 0, 256);
361
362 ahci_port_writel(ctrl, pnr, PORT_LST_ADDR, (u32)port->list);
363 ahci_port_writel(ctrl, pnr, PORT_FIS_ADDR, (u32)port->fis);
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200364 return port;
365}
366
Gerd Hoffmannce12eaf2013-09-09 16:33:06 +0200367static void ahci_port_release(struct ahci_port_s *port)
368{
369 ahci_port_reset(port->ctrl, port->pnr);
370 free(port->list);
371 free(port->fis);
372 free(port->cmd);
373 free(port);
374}
375
Gerd Hoffmannef8adc02011-08-04 19:36:31 +0200376static struct ahci_port_s* ahci_port_realloc(struct ahci_port_s *port)
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200377{
Gerd Hoffmannef8adc02011-08-04 19:36:31 +0200378 struct ahci_port_s *tmp;
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200379 u32 cmd;
380
Gerd Hoffmannef8adc02011-08-04 19:36:31 +0200381 tmp = malloc_fseg(sizeof(*port));
Gerd Hoffmannce12eaf2013-09-09 16:33:06 +0200382 if (!tmp) {
383 warn_noalloc();
384 ahci_port_release(port);
385 return NULL;
386 }
Gerd Hoffmannef8adc02011-08-04 19:36:31 +0200387 *tmp = *port;
388 free(port);
389 port = tmp;
390
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200391 ahci_port_reset(port->ctrl, port->pnr);
392
393 free(port->list);
394 free(port->fis);
395 free(port->cmd);
Gerd Hoffmann3ecdc492013-11-26 14:10:25 +0100396 port->list = memalign_high(1024, 1024);
397 port->fis = memalign_high(256, 256);
398 port->cmd = memalign_high(256, 256);
Kevin O'Connor3abdc7c2015-06-30 11:10:41 -0400399 if (!port->list || !port->fis || !port->cmd) {
400 warn_noalloc();
401 free(port->list);
402 free(port->fis);
403 free(port->cmd);
404 free(port);
405 return NULL;
406 }
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200407
408 ahci_port_writel(port->ctrl, port->pnr, PORT_LST_ADDR, (u32)port->list);
409 ahci_port_writel(port->ctrl, port->pnr, PORT_FIS_ADDR, (u32)port->fis);
410
411 cmd = ahci_port_readl(port->ctrl, port->pnr, PORT_CMD);
412 cmd |= (PORT_CMD_FIS_RX|PORT_CMD_START);
413 ahci_port_writel(port->ctrl, port->pnr, PORT_CMD, cmd);
Gerd Hoffmannef8adc02011-08-04 19:36:31 +0200414
415 return port;
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200416}
417
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200418#define MAXMODEL 40
419
420/* See ahci spec chapter 10.1 "Software Initialization of HBA" */
Kevin O'Connord83c87b2013-01-21 01:14:12 -0500421static int ahci_port_setup(struct ahci_port_s *port)
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200422{
423 struct ahci_ctrl_s *ctrl = port->ctrl;
424 u32 pnr = port->pnr;
425 char model[MAXMODEL+1];
426 u16 buffer[256];
427 u32 cmd, stat, err, tf;
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200428 int rc;
429
430 /* enable FIS recv */
431 cmd = ahci_port_readl(ctrl, pnr, PORT_CMD);
432 cmd |= PORT_CMD_FIS_RX;
433 ahci_port_writel(ctrl, pnr, PORT_CMD, cmd);
434
435 /* spin up */
436 cmd |= PORT_CMD_SPIN_UP;
437 ahci_port_writel(ctrl, pnr, PORT_CMD, cmd);
Kevin O'Connor018bdd72013-07-20 18:22:57 -0400438 u32 end = timer_calc(AHCI_LINK_TIMEOUT);
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200439 for (;;) {
440 stat = ahci_port_readl(ctrl, pnr, PORT_SCR_STAT);
441 if ((stat & 0x07) == 0x03) {
Kevin O'Connorbf079e12012-03-11 11:19:52 -0400442 dprintf(2, "AHCI/%d: link up\n", port->pnr);
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200443 break;
444 }
Kevin O'Connor018bdd72013-07-20 18:22:57 -0400445 if (timer_check(end)) {
Kevin O'Connorbf079e12012-03-11 11:19:52 -0400446 dprintf(2, "AHCI/%d: link down\n", port->pnr);
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200447 return -1;
448 }
449 yield();
450 }
451
452 /* clear error status */
453 err = ahci_port_readl(ctrl, pnr, PORT_SCR_ERR);
454 if (err)
455 ahci_port_writel(ctrl, pnr, PORT_SCR_ERR, err);
456
457 /* wait for device becoming ready */
Kevin O'Connor018bdd72013-07-20 18:22:57 -0400458 end = timer_calc(AHCI_REQUEST_TIMEOUT);
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200459 for (;;) {
460 tf = ahci_port_readl(ctrl, pnr, PORT_TFDATA);
461 if (!(tf & (ATA_CB_STAT_BSY |
462 ATA_CB_STAT_DRQ)))
463 break;
Kevin O'Connor018bdd72013-07-20 18:22:57 -0400464 if (timer_check(end)) {
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200465 warn_timeout();
466 dprintf(1, "AHCI/%d: device not ready (tf 0x%x)\n", port->pnr, tf);
467 return -1;
468 }
469 yield();
470 }
471
472 /* start device */
473 cmd |= PORT_CMD_START;
474 ahci_port_writel(ctrl, pnr, PORT_CMD, cmd);
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100475
476 sata_prep_simple(&port->cmd->fis, ATA_CMD_IDENTIFY_PACKET_DEVICE);
477 rc = ahci_command(port, 0, 0, buffer, sizeof(buffer));
478 if (rc == 0) {
479 port->atapi = 1;
480 } else {
481 port->atapi = 0;
482 sata_prep_simple(&port->cmd->fis, ATA_CMD_IDENTIFY_DEVICE);
483 rc = ahci_command(port, 0, 0, buffer, sizeof(buffer));
484 if (rc < 0)
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200485 return -1;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100486 }
487
Gerd Hoffmann0e6f6362010-12-09 08:39:48 +0100488 port->drive.cntl_id = pnr;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100489 port->drive.removable = (buffer[0] & 0x80) ? 1 : 0;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100490
491 if (!port->atapi) {
492 // found disk (ata)
Kevin O'Connorbd6afe52012-07-21 12:01:12 -0400493 port->drive.type = DTYPE_AHCI;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100494 port->drive.blksize = DISK_SECTOR_SIZE;
Kevin O'Connor8ab9a342013-09-28 23:34:49 -0400495 port->drive.pchs.cylinder = buffer[1];
496 port->drive.pchs.head = buffer[3];
497 port->drive.pchs.sector = buffer[6];
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100498
499 u64 sectors;
500 if (buffer[83] & (1 << 10)) // word 83 - lba48 support
501 sectors = *(u64*)&buffer[100]; // word 100-103
502 else
503 sectors = *(u32*)&buffer[60]; // word 60 and word 61
504 port->drive.sectors = sectors;
505 u64 adjsize = sectors >> 11;
506 char adjprefix = 'M';
507 if (adjsize >= (1 << 16)) {
508 adjsize >>= 10;
509 adjprefix = 'G';
510 }
Gerd Hoffmann2dcbf7f2011-08-04 19:36:30 +0200511 port->desc = znprintf(MAXDESCSIZE
Kevin O'Connorca2bc1c2010-12-29 21:41:19 -0500512 , "AHCI/%d: %s ATA-%d Hard-Disk (%u %ciBytes)"
513 , port->pnr
514 , ata_extract_model(model, MAXMODEL, buffer)
515 , ata_extract_version(buffer)
516 , (u32)adjsize, adjprefix);
Gerd Hoffmann2dcbf7f2011-08-04 19:36:30 +0200517 port->prio = bootprio_find_ata_device(ctrl->pci_tmp, pnr, 0);
Gerd Hoffmann3f478b92016-02-20 15:20:15 +0100518
519 s8 multi_dma = -1;
520 s8 pio_mode = -1;
521 s8 udma_mode = -1;
522 // If bit 2 in word 53 is set, udma information is valid in word 88.
523 if (buffer[53] & 0x04) {
524 udma_mode = 6;
525 while ((udma_mode >= 0) &&
526 !((buffer[88] & 0x7f) & ( 1 << udma_mode ))) {
527 udma_mode--;
528 }
529 }
530 // If bit 1 in word 53 is set, multiword-dma and advanced pio modes
531 // are available in words 63 and 64.
532 if (buffer[53] & 0x02) {
533 pio_mode = 4;
534 multi_dma = 3;
535 while ((multi_dma >= 0) &&
536 !((buffer[63] & 0x7) & ( 1 << multi_dma ))) {
537 multi_dma--;
538 }
539 while ((pio_mode >= 3) &&
540 !((buffer[64] & 0x3) & ( 1 << ( pio_mode - 3 ) ))) {
541 pio_mode--;
542 }
543 }
544 dprintf(2, "AHCI/%d: supported modes: udma %d, multi-dma %d, pio %d\n",
545 port->pnr, udma_mode, multi_dma, pio_mode);
546
547 sata_prep_simple(&port->cmd->fis, ATA_CMD_SET_FEATURES);
548 port->cmd->fis.feature = ATA_SET_FEATRUE_TRANSFER_MODE;
549 // Select used mode. UDMA first, then Multi-DMA followed by
550 // advanced PIO modes 3 or 4. If non, set default PIO.
551 if (udma_mode >= 0) {
552 dprintf(1, "AHCI/%d: Set transfer mode to UDMA-%d\n",
553 port->pnr, udma_mode);
554 port->cmd->fis.sector_count = ATA_TRANSFER_MODE_ULTRA_DMA
555 | udma_mode;
556 } else if (multi_dma >= 0) {
557 dprintf(1, "AHCI/%d: Set transfer mode to Multi-DMA-%d\n",
558 port->pnr, multi_dma);
559 port->cmd->fis.sector_count = ATA_TRANSFER_MODE_MULTIWORD_DMA
560 | multi_dma;
561 } else if (pio_mode >= 3) {
562 dprintf(1, "AHCI/%d: Set transfer mode to PIO-%d\n",
563 port->pnr, pio_mode);
564 port->cmd->fis.sector_count = ATA_TRANSFER_MODE_PIO_FLOW_CTRL
565 | pio_mode;
566 } else {
567 dprintf(1, "AHCI/%d: Set transfer mode to default PIO\n",
568 port->pnr);
569 port->cmd->fis.sector_count = ATA_TRANSFER_MODE_DEFAULT_PIO;
570 }
571 rc = ahci_command(port, 1, 0, 0, 0);
572 if (rc < 0) {
573 dprintf(1, "AHCI/%d: Set transfer mode failed.\n", port->pnr);
574 }
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100575 } else {
576 // found cdrom (atapi)
Kevin O'Connorbd6afe52012-07-21 12:01:12 -0400577 port->drive.type = DTYPE_AHCI_ATAPI;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100578 port->drive.blksize = CDROM_SECTOR_SIZE;
579 port->drive.sectors = (u64)-1;
580 u8 iscd = ((buffer[0] >> 8) & 0x1f) == 0x05;
Gerd Hoffmann263ea2f2011-08-04 19:36:29 +0200581 if (!iscd) {
Kevin O'Connorbd6afe52012-07-21 12:01:12 -0400582 dprintf(1, "AHCI/%d: atapi device isn't a cdrom\n", port->pnr);
Gerd Hoffmann263ea2f2011-08-04 19:36:29 +0200583 return -1;
584 }
Gerd Hoffmann2dcbf7f2011-08-04 19:36:30 +0200585 port->desc = znprintf(MAXDESCSIZE
Gerd Hoffmann263ea2f2011-08-04 19:36:29 +0200586 , "DVD/CD [AHCI/%d: %s ATAPI-%d DVD/CD]"
Kevin O'Connorca2bc1c2010-12-29 21:41:19 -0500587 , port->pnr
588 , ata_extract_model(model, MAXMODEL, buffer)
Gerd Hoffmann263ea2f2011-08-04 19:36:29 +0200589 , ata_extract_version(buffer));
Gerd Hoffmann2dcbf7f2011-08-04 19:36:30 +0200590 port->prio = bootprio_find_ata_device(ctrl->pci_tmp, pnr, 0);
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100591 }
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200592 return 0;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100593}
594
595// Detect any drives attached to a given controller.
596static void
Gerd Hoffmann9713f242011-08-04 19:36:28 +0200597ahci_port_detect(void *data)
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100598{
Gerd Hoffmann9713f242011-08-04 19:36:28 +0200599 struct ahci_port_s *port = data;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100600 int rc;
601
Gerd Hoffmann9713f242011-08-04 19:36:28 +0200602 dprintf(2, "AHCI/%d: probing\n", port->pnr);
603 ahci_port_reset(port->ctrl, port->pnr);
Kevin O'Connord83c87b2013-01-21 01:14:12 -0500604 rc = ahci_port_setup(port);
Gerd Hoffmann9713f242011-08-04 19:36:28 +0200605 if (rc < 0)
606 ahci_port_release(port);
Gerd Hoffmann2dcbf7f2011-08-04 19:36:30 +0200607 else {
Gerd Hoffmannef8adc02011-08-04 19:36:31 +0200608 port = ahci_port_realloc(port);
Gerd Hoffmannce12eaf2013-09-09 16:33:06 +0200609 if (port == NULL)
610 return;
Gerd Hoffmann2dcbf7f2011-08-04 19:36:30 +0200611 dprintf(1, "AHCI/%d: registering: \"%s\"\n", port->pnr, port->desc);
612 if (!port->atapi) {
613 // Register with bcv system.
614 boot_add_hd(&port->drive, port->desc, port->prio);
615 } else {
616 // fill cdidmap
617 boot_add_cd(&port->drive, port->desc, port->prio);
618 }
619 }
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100620}
621
622// Initialize an ata controller and detect its drives.
623static void
Kevin O'Connord83c87b2013-01-21 01:14:12 -0500624ahci_controller_setup(struct pci_device *pci)
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100625{
Gerd Hoffmann9713f242011-08-04 19:36:28 +0200626 struct ahci_port_s *port;
Gerd Hoffmann9713f242011-08-04 19:36:28 +0200627 u32 val, pnr, max;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100628
Kevin O'Connor6e3436b2016-02-02 22:11:30 -0500629 if (create_bounce_buf() < 0)
630 return;
631
632 void *iobase = pci_enable_membar(pci, PCI_BASE_ADDRESS_5);
633 if (!iobase)
634 return;
635
636 struct ahci_ctrl_s *ctrl = malloc_fseg(sizeof(*ctrl));
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100637 if (!ctrl) {
638 warn_noalloc();
639 return;
640 }
Scott Duplichan9c48aab2011-07-14 16:24:02 +0200641
Gerd Hoffmann9c869922011-07-14 16:24:05 +0200642 ctrl->pci_tmp = pci;
Kevin O'Connor6e3436b2016-02-02 22:11:30 -0500643 ctrl->iobase = iobase;
644 ctrl->irq = pci_config_readb(pci->bdf, PCI_INTERRUPT_LINE);
Kevin O'Connor7b673002016-02-03 03:03:15 -0500645 dprintf(1, "AHCI controller at %pP, iobase %p, irq %d\n"
646 , pci, ctrl->iobase, ctrl->irq);
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100647
Kevin O'Connor6e3436b2016-02-02 22:11:30 -0500648 pci_enable_busmaster(pci);
Kevin O'Connor7eb02222010-12-12 14:01:47 -0500649
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100650 val = ahci_ctrl_readl(ctrl, HOST_CTL);
651 ahci_ctrl_writel(ctrl, HOST_CTL, val | HOST_CTL_AHCI_EN);
652
653 ctrl->caps = ahci_ctrl_readl(ctrl, HOST_CAP);
654 ctrl->ports = ahci_ctrl_readl(ctrl, HOST_PORTS_IMPL);
655 dprintf(2, "AHCI: cap 0x%x, ports_impl 0x%x\n",
656 ctrl->caps, ctrl->ports);
657
Vladimir Serbinenko40dfc0e2015-05-18 15:00:31 +0200658 max = 0x1f;
Gerd Hoffmann9713f242011-08-04 19:36:28 +0200659 for (pnr = 0; pnr <= max; pnr++) {
660 if (!(ctrl->ports & (1 << pnr)))
661 continue;
662 port = ahci_port_alloc(ctrl, pnr);
663 if (port == NULL)
664 continue;
665 run_thread(ahci_port_detect, port);
666 }
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100667}
668
669// Locate and init ahci controllers.
670static void
Kevin O'Connord83c87b2013-01-21 01:14:12 -0500671ahci_scan(void)
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100672{
673 // Scan PCI bus for ATA adapters
Kevin O'Connor9cb49922011-06-20 22:22:42 -0400674 struct pci_device *pci;
675 foreachpci(pci) {
676 if (pci->class != PCI_CLASS_STORAGE_SATA)
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100677 continue;
Kevin O'Connor9cb49922011-06-20 22:22:42 -0400678 if (pci->prog_if != 1 /* AHCI rev 1 */)
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100679 continue;
Kevin O'Connord83c87b2013-01-21 01:14:12 -0500680 ahci_controller_setup(pci);
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100681 }
682}
683
684void
685ahci_setup(void)
686{
687 ASSERT32FLAT();
688 if (!CONFIG_AHCI)
689 return;
690
691 dprintf(3, "init ahci\n");
Kevin O'Connord83c87b2013-01-21 01:14:12 -0500692 ahci_scan();
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100693}