blob: 726fe25eed1da6463186a183c838d3bf7f3ea8c8 [file] [log] [blame]
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +01001// Low level AHCI disk access
2//
3// Copyright (C) 2010 Gerd Hoffmann <kraxel@redhat.com>
4//
5// This file may be distributed under the terms of the GNU LGPLv3 license.
6
Kevin O'Connor2d2fa312013-09-14 21:55:26 -04007#include "ahci.h" // CDB_CMD_READ_10
8#include "ata.h" // ATA_CB_STAT
Kevin O'Connor4bc49972012-05-13 22:58:08 -04009#include "biosvar.h" // GET_GLOBAL
Kevin O'Connor2d2fa312013-09-14 21:55:26 -040010#include "blockcmd.h" // CDB_CMD_READ_10
Kevin O'Connor2d2fa312013-09-14 21:55:26 -040011#include "malloc.h" // free
12#include "output.h" // dprintf
Kevin O'Connor9cb49922011-06-20 22:22:42 -040013#include "pci.h" // foreachpci
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +010014#include "pci_ids.h" // PCI_CLASS_STORAGE_OTHER
15#include "pci_regs.h" // PCI_INTERRUPT_LINE
Kevin O'Connor3df600b2013-09-14 19:28:55 -040016#include "stacks.h" // yield
Kevin O'Connor135f3f62013-09-14 23:57:26 -040017#include "std/disk.h" // DISK_RET_SUCCESS
Kevin O'Connorfa9c66a2013-09-14 19:10:40 -040018#include "string.h" // memset
Kevin O'Connor2d2fa312013-09-14 21:55:26 -040019#include "util.h" // timer_calc
Kevin O'Connor4ade5232013-09-18 21:41:48 -040020#include "x86.h" // inb
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +010021
Gerd Hoffmanne1041192011-07-14 16:24:04 +020022#define AHCI_REQUEST_TIMEOUT 32000 // 32 seconds max for IDE ops
23#define AHCI_RESET_TIMEOUT 500 // 500 miliseconds
24#define AHCI_LINK_TIMEOUT 10 // 10 miliseconds
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +010025
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +010026// prepare sata command fis
27static void sata_prep_simple(struct sata_cmd_fis *fis, u8 command)
28{
29 memset_fl(fis, 0, sizeof(*fis));
Gerd Hoffmann26ae90b2013-11-26 14:01:20 +010030 fis->command = command;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +010031}
32
33static void sata_prep_readwrite(struct sata_cmd_fis *fis,
34 struct disk_op_s *op, int iswrite)
35{
36 u64 lba = op->lba;
37 u8 command;
38
39 memset_fl(fis, 0, sizeof(*fis));
40
41 if (op->count >= (1<<8) || lba + op->count >= (1<<28)) {
Gerd Hoffmann26ae90b2013-11-26 14:01:20 +010042 fis->sector_count2 = op->count >> 8;
43 fis->lba_low2 = lba >> 24;
44 fis->lba_mid2 = lba >> 32;
45 fis->lba_high2 = lba >> 40;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +010046 lba &= 0xffffff;
47 command = (iswrite ? ATA_CMD_WRITE_DMA_EXT
48 : ATA_CMD_READ_DMA_EXT);
49 } else {
50 command = (iswrite ? ATA_CMD_WRITE_DMA
51 : ATA_CMD_READ_DMA);
52 }
Gerd Hoffmann26ae90b2013-11-26 14:01:20 +010053 fis->feature = 1; /* dma */
54 fis->command = command;
55 fis->sector_count = op->count;
56 fis->lba_low = lba;
57 fis->lba_mid = lba >> 8;
58 fis->lba_high = lba >> 16;
59 fis->device = ((lba >> 24) & 0xf) | ATA_CB_DH_LBA;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +010060}
61
62static void sata_prep_atapi(struct sata_cmd_fis *fis, u16 blocksize)
63{
64 memset_fl(fis, 0, sizeof(*fis));
Gerd Hoffmann26ae90b2013-11-26 14:01:20 +010065 fis->command = ATA_CMD_PACKET;
66 fis->feature = 1; /* dma */
67 fis->lba_mid = blocksize;
68 fis->lba_high = blocksize >> 8;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +010069}
70
71// ahci register access helpers
72static u32 ahci_ctrl_readl(struct ahci_ctrl_s *ctrl, u32 reg)
73{
Gerd Hoffmann26ae90b2013-11-26 14:01:20 +010074 u32 addr = ctrl->iobase + reg;
Kevin O'Connor7c80bb82013-10-02 21:28:08 -040075 return readl((void*)addr);
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +010076}
77
78static void ahci_ctrl_writel(struct ahci_ctrl_s *ctrl, u32 reg, u32 val)
79{
Gerd Hoffmann26ae90b2013-11-26 14:01:20 +010080 u32 addr = ctrl->iobase + reg;
Kevin O'Connor7c80bb82013-10-02 21:28:08 -040081 writel((void*)addr, val);
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +010082}
83
84static u32 ahci_port_to_ctrl(u32 pnr, u32 port_reg)
85{
86 u32 ctrl_reg = 0x100;
87 ctrl_reg += pnr * 0x80;
88 ctrl_reg += port_reg;
89 return ctrl_reg;
90}
91
92static u32 ahci_port_readl(struct ahci_ctrl_s *ctrl, u32 pnr, u32 reg)
93{
94 u32 ctrl_reg = ahci_port_to_ctrl(pnr, reg);
95 return ahci_ctrl_readl(ctrl, ctrl_reg);
96}
97
98static void ahci_port_writel(struct ahci_ctrl_s *ctrl, u32 pnr, u32 reg, u32 val)
99{
100 u32 ctrl_reg = ahci_port_to_ctrl(pnr, reg);
101 ahci_ctrl_writel(ctrl, ctrl_reg, val);
102}
103
104// submit ahci command + wait for result
Kevin O'Connor1902c942013-10-26 11:48:06 -0400105static int ahci_command(struct ahci_port_s *port_gf, int iswrite, int isatapi,
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100106 void *buffer, u32 bsize)
107{
Gerd Hoffmann6f850492011-07-14 16:24:01 +0200108 u32 val, status, success, flags, intbits, error;
Gerd Hoffmann26ae90b2013-11-26 14:01:20 +0100109 struct ahci_ctrl_s *ctrl = port_gf->ctrl;
110 struct ahci_cmd_s *cmd = port_gf->cmd;
111 struct ahci_fis_s *fis = port_gf->fis;
112 struct ahci_list_s *list = port_gf->list;
113 u32 pnr = port_gf->pnr;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100114
Gerd Hoffmann26ae90b2013-11-26 14:01:20 +0100115 cmd->fis.reg = 0x27;
Gerd Hoffmann4a0f3662013-11-26 14:02:54 +0100116 cmd->fis.pmp_type = 1 << 7; /* cmd fis */
117 cmd->prdt[0].base = (u32)buffer;
Gerd Hoffmann26ae90b2013-11-26 14:01:20 +0100118 cmd->prdt[0].baseu = 0;
119 cmd->prdt[0].flags = bsize-1;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100120
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100121 flags = ((1 << 16) | /* one prd entry */
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100122 (iswrite ? (1 << 6) : 0) |
123 (isatapi ? (1 << 5) : 0) |
Gerd Hoffmanna8c6a4e2011-07-14 16:23:59 +0200124 (5 << 0)); /* fis length (dwords) */
Gerd Hoffmann26ae90b2013-11-26 14:01:20 +0100125 list[0].flags = flags;
126 list[0].bytes = 0;
Gerd Hoffmann4a0f3662013-11-26 14:02:54 +0100127 list[0].base = (u32)(cmd);
Gerd Hoffmann26ae90b2013-11-26 14:01:20 +0100128 list[0].baseu = 0;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100129
Kevin O'Connorbf079e12012-03-11 11:19:52 -0400130 dprintf(8, "AHCI/%d: send cmd ...\n", pnr);
Gerd Hoffmann07532972011-07-14 16:24:00 +0200131 intbits = ahci_port_readl(ctrl, pnr, PORT_IRQ_STAT);
132 if (intbits)
133 ahci_port_writel(ctrl, pnr, PORT_IRQ_STAT, intbits);
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100134 ahci_port_writel(ctrl, pnr, PORT_SCR_ACT, 1);
135 ahci_port_writel(ctrl, pnr, PORT_CMD_ISSUE, 1);
Gerd Hoffmann07532972011-07-14 16:24:00 +0200136
Kevin O'Connor018bdd72013-07-20 18:22:57 -0400137 u32 end = timer_calc(AHCI_REQUEST_TIMEOUT);
Gerd Hoffmann07532972011-07-14 16:24:00 +0200138 do {
139 for (;;) {
140 intbits = ahci_port_readl(ctrl, pnr, PORT_IRQ_STAT);
141 if (intbits) {
142 ahci_port_writel(ctrl, pnr, PORT_IRQ_STAT, intbits);
143 if (intbits & 0x02) {
Kevin O'Connor890c0852012-05-24 23:55:00 -0400144 status = GET_LOWFLAT(fis->psfis[2]);
145 error = GET_LOWFLAT(fis->psfis[3]);
Gerd Hoffmann07532972011-07-14 16:24:00 +0200146 break;
147 }
148 if (intbits & 0x01) {
Kevin O'Connor890c0852012-05-24 23:55:00 -0400149 status = GET_LOWFLAT(fis->rfis[2]);
150 error = GET_LOWFLAT(fis->rfis[3]);
Gerd Hoffmann07532972011-07-14 16:24:00 +0200151 break;
152 }
153 }
Kevin O'Connor018bdd72013-07-20 18:22:57 -0400154 if (timer_check(end)) {
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200155 warn_timeout();
156 return -1;
157 }
Gerd Hoffmann07532972011-07-14 16:24:00 +0200158 yield();
159 }
Kevin O'Connorbf079e12012-03-11 11:19:52 -0400160 dprintf(8, "AHCI/%d: ... intbits 0x%x, status 0x%x ...\n",
Gerd Hoffmann07532972011-07-14 16:24:00 +0200161 pnr, intbits, status);
162 } while (status & ATA_CB_STAT_BSY);
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100163
164 success = (0x00 == (status & (ATA_CB_STAT_BSY | ATA_CB_STAT_DF |
Gerd Hoffmanncbda7952011-07-14 16:24:03 +0200165 ATA_CB_STAT_ERR)) &&
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100166 ATA_CB_STAT_RDY == (status & (ATA_CB_STAT_RDY)));
Gerd Hoffmann6f850492011-07-14 16:24:01 +0200167 if (success) {
Kevin O'Connorbf079e12012-03-11 11:19:52 -0400168 dprintf(8, "AHCI/%d: ... finished, status 0x%x, OK\n", pnr,
Gerd Hoffmann6f850492011-07-14 16:24:01 +0200169 status);
170 } else {
171 dprintf(2, "AHCI/%d: ... finished, status 0x%x, ERROR 0x%x\n", pnr,
172 status, error);
173
174 // non-queued error recovery (AHCI 1.3 section 6.2.2.1)
175 // Clears PxCMD.ST to 0 to reset the PxCI register
176 val = ahci_port_readl(ctrl, pnr, PORT_CMD);
177 ahci_port_writel(ctrl, pnr, PORT_CMD, val & ~PORT_CMD_START);
178
179 // waits for PxCMD.CR to clear to 0
180 while (1) {
181 val = ahci_port_readl(ctrl, pnr, PORT_CMD);
182 if ((val & PORT_CMD_LIST_ON) == 0)
183 break;
184 yield();
185 }
186
187 // Clears any error bits in PxSERR to enable capturing new errors
188 val = ahci_port_readl(ctrl, pnr, PORT_SCR_ERR);
189 ahci_port_writel(ctrl, pnr, PORT_SCR_ERR, val);
190
191 // Clears status bits in PxIS as appropriate
192 val = ahci_port_readl(ctrl, pnr, PORT_IRQ_STAT);
193 ahci_port_writel(ctrl, pnr, PORT_IRQ_STAT, val);
194
195 // If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to 1, issue
196 // a COMRESET to the device to put it in an idle state
197 val = ahci_port_readl(ctrl, pnr, PORT_TFDATA);
198 if (val & (ATA_CB_STAT_BSY | ATA_CB_STAT_DRQ)) {
199 dprintf(2, "AHCI/%d: issue comreset\n", pnr);
200 val = ahci_port_readl(ctrl, pnr, PORT_SCR_CTL);
201 // set Device Detection Initialization (DET) to 1 for 1 ms for comreset
202 ahci_port_writel(ctrl, pnr, PORT_SCR_CTL, val | 1);
203 mdelay (1);
204 ahci_port_writel(ctrl, pnr, PORT_SCR_CTL, val);
205 }
206
207 // Sets PxCMD.ST to 1 to enable issuing new commands
208 val = ahci_port_readl(ctrl, pnr, PORT_CMD);
209 ahci_port_writel(ctrl, pnr, PORT_CMD, val | PORT_CMD_START);
210 }
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100211 return success ? 0 : -1;
212}
213
214#define CDROM_CDB_SIZE 12
215
Kevin O'Connor0e5c7702015-07-07 11:24:27 -0400216int ahci_atapi_process_op(struct disk_op_s *op)
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100217{
Kevin O'Connor80c2b6e2010-12-05 12:52:02 -0500218 if (! CONFIG_AHCI)
219 return 0;
220
Kevin O'Connor1902c942013-10-26 11:48:06 -0400221 struct ahci_port_s *port_gf = container_of(
222 op->drive_gf, struct ahci_port_s, drive);
Gerd Hoffmann26ae90b2013-11-26 14:01:20 +0100223 struct ahci_cmd_s *cmd = port_gf->cmd;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100224
Kevin O'Connor0e5c7702015-07-07 11:24:27 -0400225 if (op->command == CMD_WRITE || op->command == CMD_FORMAT)
226 return DISK_RET_EWRITEPROTECT;
227 int blocksize = scsi_fill_cmd(op, cmd->atapi, CDROM_CDB_SIZE);
228 if (blocksize < 0)
229 return default_process_op(op);
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100230 sata_prep_atapi(&cmd->fis, blocksize);
Kevin O'Connor0e5c7702015-07-07 11:24:27 -0400231 int rc = ahci_command(port_gf, 0, 1, op->buf_fl, op->count * blocksize);
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100232 if (rc < 0)
233 return DISK_RET_EBADTRACK;
234 return DISK_RET_SUCCESS;
235}
236
Scott Duplichan9c48aab2011-07-14 16:24:02 +0200237// read/write count blocks from a harddrive, op->buf_fl must be word aligned
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100238static int
Scott Duplichan9c48aab2011-07-14 16:24:02 +0200239ahci_disk_readwrite_aligned(struct disk_op_s *op, int iswrite)
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100240{
Kevin O'Connor1902c942013-10-26 11:48:06 -0400241 struct ahci_port_s *port_gf = container_of(
242 op->drive_gf, struct ahci_port_s, drive);
Gerd Hoffmann26ae90b2013-11-26 14:01:20 +0100243 struct ahci_cmd_s *cmd = port_gf->cmd;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100244 int rc;
245
246 sata_prep_readwrite(&cmd->fis, op, iswrite);
Kevin O'Connor1902c942013-10-26 11:48:06 -0400247 rc = ahci_command(port_gf, iswrite, 0, op->buf_fl,
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100248 op->count * DISK_SECTOR_SIZE);
Kevin O'Connorbf079e12012-03-11 11:19:52 -0400249 dprintf(8, "ahci disk %s, lba %6x, count %3x, buf %p, rc %d\n",
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100250 iswrite ? "write" : "read", (u32)op->lba, op->count, op->buf_fl, rc);
251 if (rc < 0)
252 return DISK_RET_EBADTRACK;
253 return DISK_RET_SUCCESS;
254}
255
Scott Duplichan9c48aab2011-07-14 16:24:02 +0200256// read/write count blocks from a harddrive.
257static int
258ahci_disk_readwrite(struct disk_op_s *op, int iswrite)
259{
260 // if caller's buffer is word aligned, use it directly
261 if (((u32) op->buf_fl & 1) == 0)
262 return ahci_disk_readwrite_aligned(op, iswrite);
263
264 // Use a word aligned buffer for AHCI I/O
265 int rc;
266 struct disk_op_s localop = *op;
Gerd Hoffmann26ae90b2013-11-26 14:01:20 +0100267 u8 *alignedbuf_fl = bounce_buf_fl;
Scott Duplichan9c48aab2011-07-14 16:24:02 +0200268 u8 *position = op->buf_fl;
269
270 localop.buf_fl = alignedbuf_fl;
271 localop.count = 1;
272
273 if (iswrite) {
274 u16 block;
275 for (block = 0; block < op->count; block++) {
276 memcpy_fl (alignedbuf_fl, position, DISK_SECTOR_SIZE);
277 rc = ahci_disk_readwrite_aligned (&localop, 1);
278 if (rc)
279 return rc;
280 position += DISK_SECTOR_SIZE;
281 localop.lba++;
282 }
283 } else { // read
284 u16 block;
285 for (block = 0; block < op->count; block++) {
286 rc = ahci_disk_readwrite_aligned (&localop, 0);
287 if (rc)
288 return rc;
289 memcpy_fl (position, alignedbuf_fl, DISK_SECTOR_SIZE);
290 position += DISK_SECTOR_SIZE;
291 localop.lba++;
292 }
293 }
294 return DISK_RET_SUCCESS;
295}
296
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100297// command demuxer
Kevin O'Connorc7fa7892015-07-07 08:35:51 -0400298int
Kevin O'Connor7c80bb82013-10-02 21:28:08 -0400299process_ahci_op(struct disk_op_s *op)
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100300{
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100301 if (!CONFIG_AHCI)
302 return 0;
Kevin O'Connorbd6afe52012-07-21 12:01:12 -0400303 switch (op->command) {
304 case CMD_READ:
305 return ahci_disk_readwrite(op, 0);
306 case CMD_WRITE:
307 return ahci_disk_readwrite(op, 1);
Kevin O'Connorbd6afe52012-07-21 12:01:12 -0400308 default:
Kevin O'Connor85c72c62015-07-07 09:01:52 -0400309 return default_process_op(op);
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100310 }
311}
312
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100313static void
314ahci_port_reset(struct ahci_ctrl_s *ctrl, u32 pnr)
315{
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200316 u32 val;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100317
318 /* disable FIS + CMD */
Kevin O'Connor018bdd72013-07-20 18:22:57 -0400319 u32 end = timer_calc(AHCI_RESET_TIMEOUT);
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200320 for (;;) {
321 val = ahci_port_readl(ctrl, pnr, PORT_CMD);
322 if (!(val & (PORT_CMD_FIS_RX | PORT_CMD_START |
323 PORT_CMD_FIS_ON | PORT_CMD_LIST_ON)))
324 break;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100325 val &= ~(PORT_CMD_FIS_RX | PORT_CMD_START);
326 ahci_port_writel(ctrl, pnr, PORT_CMD, val);
Kevin O'Connor018bdd72013-07-20 18:22:57 -0400327 if (timer_check(end)) {
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200328 warn_timeout();
329 break;
330 }
331 yield();
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100332 }
333
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100334 /* disable + clear IRQs */
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200335 ahci_port_writel(ctrl, pnr, PORT_IRQ_MASK, 0);
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100336 val = ahci_port_readl(ctrl, pnr, PORT_IRQ_STAT);
337 if (val)
338 ahci_port_writel(ctrl, pnr, PORT_IRQ_STAT, val);
339}
340
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100341static struct ahci_port_s*
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200342ahci_port_alloc(struct ahci_ctrl_s *ctrl, u32 pnr)
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100343{
Gerd Hoffmannef8adc02011-08-04 19:36:31 +0200344 struct ahci_port_s *port = malloc_tmp(sizeof(*port));
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100345
346 if (!port) {
347 warn_noalloc();
348 return NULL;
349 }
350 port->pnr = pnr;
351 port->ctrl = ctrl;
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200352 port->list = memalign_tmp(1024, 1024);
353 port->fis = memalign_tmp(256, 256);
354 port->cmd = memalign_tmp(256, 256);
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100355 if (port->list == NULL || port->fis == NULL || port->cmd == NULL) {
356 warn_noalloc();
357 return NULL;
358 }
359 memset(port->list, 0, 1024);
360 memset(port->fis, 0, 256);
361 memset(port->cmd, 0, 256);
362
363 ahci_port_writel(ctrl, pnr, PORT_LST_ADDR, (u32)port->list);
364 ahci_port_writel(ctrl, pnr, PORT_FIS_ADDR, (u32)port->fis);
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200365 return port;
366}
367
Gerd Hoffmannce12eaf2013-09-09 16:33:06 +0200368static void ahci_port_release(struct ahci_port_s *port)
369{
370 ahci_port_reset(port->ctrl, port->pnr);
371 free(port->list);
372 free(port->fis);
373 free(port->cmd);
374 free(port);
375}
376
Gerd Hoffmannef8adc02011-08-04 19:36:31 +0200377static struct ahci_port_s* ahci_port_realloc(struct ahci_port_s *port)
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200378{
Gerd Hoffmannef8adc02011-08-04 19:36:31 +0200379 struct ahci_port_s *tmp;
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200380 u32 cmd;
381
Gerd Hoffmannef8adc02011-08-04 19:36:31 +0200382 tmp = malloc_fseg(sizeof(*port));
Gerd Hoffmannce12eaf2013-09-09 16:33:06 +0200383 if (!tmp) {
384 warn_noalloc();
385 ahci_port_release(port);
386 return NULL;
387 }
Gerd Hoffmannef8adc02011-08-04 19:36:31 +0200388 *tmp = *port;
389 free(port);
390 port = tmp;
391
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200392 ahci_port_reset(port->ctrl, port->pnr);
393
394 free(port->list);
395 free(port->fis);
396 free(port->cmd);
Gerd Hoffmann3ecdc492013-11-26 14:10:25 +0100397 port->list = memalign_high(1024, 1024);
398 port->fis = memalign_high(256, 256);
399 port->cmd = memalign_high(256, 256);
Kevin O'Connor3abdc7c2015-06-30 11:10:41 -0400400 if (!port->list || !port->fis || !port->cmd) {
401 warn_noalloc();
402 free(port->list);
403 free(port->fis);
404 free(port->cmd);
405 free(port);
406 return NULL;
407 }
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200408
409 ahci_port_writel(port->ctrl, port->pnr, PORT_LST_ADDR, (u32)port->list);
410 ahci_port_writel(port->ctrl, port->pnr, PORT_FIS_ADDR, (u32)port->fis);
411
412 cmd = ahci_port_readl(port->ctrl, port->pnr, PORT_CMD);
413 cmd |= (PORT_CMD_FIS_RX|PORT_CMD_START);
414 ahci_port_writel(port->ctrl, port->pnr, PORT_CMD, cmd);
Gerd Hoffmannef8adc02011-08-04 19:36:31 +0200415
416 return port;
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200417}
418
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200419#define MAXMODEL 40
420
421/* See ahci spec chapter 10.1 "Software Initialization of HBA" */
Kevin O'Connord83c87b2013-01-21 01:14:12 -0500422static int ahci_port_setup(struct ahci_port_s *port)
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200423{
424 struct ahci_ctrl_s *ctrl = port->ctrl;
425 u32 pnr = port->pnr;
426 char model[MAXMODEL+1];
427 u16 buffer[256];
428 u32 cmd, stat, err, tf;
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200429 int rc;
430
431 /* enable FIS recv */
432 cmd = ahci_port_readl(ctrl, pnr, PORT_CMD);
433 cmd |= PORT_CMD_FIS_RX;
434 ahci_port_writel(ctrl, pnr, PORT_CMD, cmd);
435
436 /* spin up */
437 cmd |= PORT_CMD_SPIN_UP;
438 ahci_port_writel(ctrl, pnr, PORT_CMD, cmd);
Kevin O'Connor018bdd72013-07-20 18:22:57 -0400439 u32 end = timer_calc(AHCI_LINK_TIMEOUT);
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200440 for (;;) {
441 stat = ahci_port_readl(ctrl, pnr, PORT_SCR_STAT);
442 if ((stat & 0x07) == 0x03) {
Kevin O'Connorbf079e12012-03-11 11:19:52 -0400443 dprintf(2, "AHCI/%d: link up\n", port->pnr);
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200444 break;
445 }
Kevin O'Connor018bdd72013-07-20 18:22:57 -0400446 if (timer_check(end)) {
Kevin O'Connorbf079e12012-03-11 11:19:52 -0400447 dprintf(2, "AHCI/%d: link down\n", port->pnr);
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200448 return -1;
449 }
450 yield();
451 }
452
453 /* clear error status */
454 err = ahci_port_readl(ctrl, pnr, PORT_SCR_ERR);
455 if (err)
456 ahci_port_writel(ctrl, pnr, PORT_SCR_ERR, err);
457
458 /* wait for device becoming ready */
Kevin O'Connor018bdd72013-07-20 18:22:57 -0400459 end = timer_calc(AHCI_REQUEST_TIMEOUT);
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200460 for (;;) {
461 tf = ahci_port_readl(ctrl, pnr, PORT_TFDATA);
462 if (!(tf & (ATA_CB_STAT_BSY |
463 ATA_CB_STAT_DRQ)))
464 break;
Kevin O'Connor018bdd72013-07-20 18:22:57 -0400465 if (timer_check(end)) {
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200466 warn_timeout();
467 dprintf(1, "AHCI/%d: device not ready (tf 0x%x)\n", port->pnr, tf);
468 return -1;
469 }
470 yield();
471 }
472
473 /* start device */
474 cmd |= PORT_CMD_START;
475 ahci_port_writel(ctrl, pnr, PORT_CMD, cmd);
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100476
477 sata_prep_simple(&port->cmd->fis, ATA_CMD_IDENTIFY_PACKET_DEVICE);
478 rc = ahci_command(port, 0, 0, buffer, sizeof(buffer));
479 if (rc == 0) {
480 port->atapi = 1;
481 } else {
482 port->atapi = 0;
483 sata_prep_simple(&port->cmd->fis, ATA_CMD_IDENTIFY_DEVICE);
484 rc = ahci_command(port, 0, 0, buffer, sizeof(buffer));
485 if (rc < 0)
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200486 return -1;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100487 }
488
Gerd Hoffmann0e6f6362010-12-09 08:39:48 +0100489 port->drive.cntl_id = pnr;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100490 port->drive.removable = (buffer[0] & 0x80) ? 1 : 0;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100491
492 if (!port->atapi) {
493 // found disk (ata)
Kevin O'Connorbd6afe52012-07-21 12:01:12 -0400494 port->drive.type = DTYPE_AHCI;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100495 port->drive.blksize = DISK_SECTOR_SIZE;
Kevin O'Connor8ab9a342013-09-28 23:34:49 -0400496 port->drive.pchs.cylinder = buffer[1];
497 port->drive.pchs.head = buffer[3];
498 port->drive.pchs.sector = buffer[6];
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100499
500 u64 sectors;
501 if (buffer[83] & (1 << 10)) // word 83 - lba48 support
502 sectors = *(u64*)&buffer[100]; // word 100-103
503 else
504 sectors = *(u32*)&buffer[60]; // word 60 and word 61
505 port->drive.sectors = sectors;
506 u64 adjsize = sectors >> 11;
507 char adjprefix = 'M';
508 if (adjsize >= (1 << 16)) {
509 adjsize >>= 10;
510 adjprefix = 'G';
511 }
Gerd Hoffmann2dcbf7f2011-08-04 19:36:30 +0200512 port->desc = znprintf(MAXDESCSIZE
Kevin O'Connorca2bc1c2010-12-29 21:41:19 -0500513 , "AHCI/%d: %s ATA-%d Hard-Disk (%u %ciBytes)"
514 , port->pnr
515 , ata_extract_model(model, MAXMODEL, buffer)
516 , ata_extract_version(buffer)
517 , (u32)adjsize, adjprefix);
Gerd Hoffmann2dcbf7f2011-08-04 19:36:30 +0200518 port->prio = bootprio_find_ata_device(ctrl->pci_tmp, pnr, 0);
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100519 } else {
520 // found cdrom (atapi)
Kevin O'Connorbd6afe52012-07-21 12:01:12 -0400521 port->drive.type = DTYPE_AHCI_ATAPI;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100522 port->drive.blksize = CDROM_SECTOR_SIZE;
523 port->drive.sectors = (u64)-1;
524 u8 iscd = ((buffer[0] >> 8) & 0x1f) == 0x05;
Gerd Hoffmann263ea2f2011-08-04 19:36:29 +0200525 if (!iscd) {
Kevin O'Connorbd6afe52012-07-21 12:01:12 -0400526 dprintf(1, "AHCI/%d: atapi device isn't a cdrom\n", port->pnr);
Gerd Hoffmann263ea2f2011-08-04 19:36:29 +0200527 return -1;
528 }
Gerd Hoffmann2dcbf7f2011-08-04 19:36:30 +0200529 port->desc = znprintf(MAXDESCSIZE
Gerd Hoffmann263ea2f2011-08-04 19:36:29 +0200530 , "DVD/CD [AHCI/%d: %s ATAPI-%d DVD/CD]"
Kevin O'Connorca2bc1c2010-12-29 21:41:19 -0500531 , port->pnr
532 , ata_extract_model(model, MAXMODEL, buffer)
Gerd Hoffmann263ea2f2011-08-04 19:36:29 +0200533 , ata_extract_version(buffer));
Gerd Hoffmann2dcbf7f2011-08-04 19:36:30 +0200534 port->prio = bootprio_find_ata_device(ctrl->pci_tmp, pnr, 0);
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100535 }
Gerd Hoffmanne1041192011-07-14 16:24:04 +0200536 return 0;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100537}
538
539// Detect any drives attached to a given controller.
540static void
Gerd Hoffmann9713f242011-08-04 19:36:28 +0200541ahci_port_detect(void *data)
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100542{
Gerd Hoffmann9713f242011-08-04 19:36:28 +0200543 struct ahci_port_s *port = data;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100544 int rc;
545
Gerd Hoffmann9713f242011-08-04 19:36:28 +0200546 dprintf(2, "AHCI/%d: probing\n", port->pnr);
547 ahci_port_reset(port->ctrl, port->pnr);
Kevin O'Connord83c87b2013-01-21 01:14:12 -0500548 rc = ahci_port_setup(port);
Gerd Hoffmann9713f242011-08-04 19:36:28 +0200549 if (rc < 0)
550 ahci_port_release(port);
Gerd Hoffmann2dcbf7f2011-08-04 19:36:30 +0200551 else {
Gerd Hoffmannef8adc02011-08-04 19:36:31 +0200552 port = ahci_port_realloc(port);
Gerd Hoffmannce12eaf2013-09-09 16:33:06 +0200553 if (port == NULL)
554 return;
Gerd Hoffmann2dcbf7f2011-08-04 19:36:30 +0200555 dprintf(1, "AHCI/%d: registering: \"%s\"\n", port->pnr, port->desc);
556 if (!port->atapi) {
557 // Register with bcv system.
558 boot_add_hd(&port->drive, port->desc, port->prio);
559 } else {
560 // fill cdidmap
561 boot_add_cd(&port->drive, port->desc, port->prio);
562 }
563 }
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100564}
565
566// Initialize an ata controller and detect its drives.
567static void
Kevin O'Connord83c87b2013-01-21 01:14:12 -0500568ahci_controller_setup(struct pci_device *pci)
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100569{
570 struct ahci_ctrl_s *ctrl = malloc_fseg(sizeof(*ctrl));
Gerd Hoffmann9713f242011-08-04 19:36:28 +0200571 struct ahci_port_s *port;
Gerd Hoffmann9c869922011-07-14 16:24:05 +0200572 u16 bdf = pci->bdf;
Gerd Hoffmann9713f242011-08-04 19:36:28 +0200573 u32 val, pnr, max;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100574
575 if (!ctrl) {
576 warn_noalloc();
577 return;
578 }
Scott Duplichan9c48aab2011-07-14 16:24:02 +0200579
Kevin O'Connord83c87b2013-01-21 01:14:12 -0500580 if (create_bounce_buf() < 0) {
Scott Duplichan9c48aab2011-07-14 16:24:02 +0200581 warn_noalloc();
Gerd Hoffmannd7a7cf32011-08-04 19:36:27 +0200582 free(ctrl);
Scott Duplichan9c48aab2011-07-14 16:24:02 +0200583 return;
584 }
585
Gerd Hoffmann9c869922011-07-14 16:24:05 +0200586 ctrl->pci_tmp = pci;
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100587 ctrl->pci_bdf = bdf;
588 ctrl->iobase = pci_config_readl(bdf, PCI_BASE_ADDRESS_5);
589 ctrl->irq = pci_config_readb(bdf, PCI_INTERRUPT_LINE);
590 dprintf(1, "AHCI controller at %02x.%x, iobase %x, irq %d\n",
591 bdf >> 3, bdf & 7, ctrl->iobase, ctrl->irq);
592
Kevin O'Connor7eb02222010-12-12 14:01:47 -0500593 pci_config_maskw(bdf, PCI_COMMAND, 0,
594 PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER);
595
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100596 val = ahci_ctrl_readl(ctrl, HOST_CTL);
597 ahci_ctrl_writel(ctrl, HOST_CTL, val | HOST_CTL_AHCI_EN);
598
599 ctrl->caps = ahci_ctrl_readl(ctrl, HOST_CAP);
600 ctrl->ports = ahci_ctrl_readl(ctrl, HOST_PORTS_IMPL);
601 dprintf(2, "AHCI: cap 0x%x, ports_impl 0x%x\n",
602 ctrl->caps, ctrl->ports);
603
Vladimir Serbinenko40dfc0e2015-05-18 15:00:31 +0200604 max = 0x1f;
Gerd Hoffmann9713f242011-08-04 19:36:28 +0200605 for (pnr = 0; pnr <= max; pnr++) {
606 if (!(ctrl->ports & (1 << pnr)))
607 continue;
608 port = ahci_port_alloc(ctrl, pnr);
609 if (port == NULL)
610 continue;
611 run_thread(ahci_port_detect, port);
612 }
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100613}
614
615// Locate and init ahci controllers.
616static void
Kevin O'Connord83c87b2013-01-21 01:14:12 -0500617ahci_scan(void)
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100618{
619 // Scan PCI bus for ATA adapters
Kevin O'Connor9cb49922011-06-20 22:22:42 -0400620 struct pci_device *pci;
621 foreachpci(pci) {
622 if (pci->class != PCI_CLASS_STORAGE_SATA)
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100623 continue;
Kevin O'Connor9cb49922011-06-20 22:22:42 -0400624 if (pci->prog_if != 1 /* AHCI rev 1 */)
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100625 continue;
Kevin O'Connord83c87b2013-01-21 01:14:12 -0500626 ahci_controller_setup(pci);
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100627 }
628}
629
630void
631ahci_setup(void)
632{
633 ASSERT32FLAT();
634 if (!CONFIG_AHCI)
635 return;
636
637 dprintf(3, "init ahci\n");
Kevin O'Connord83c87b2013-01-21 01:14:12 -0500638 ahci_scan();
Gerd Hoffmannd52fdf62010-11-29 09:42:13 +0100639}