blob: 14f243cd697a861c8fb5872f8e47a2f7302dd90d [file] [log] [blame]
Gerd Hoffmanne144bb72013-06-03 16:30:18 +02001#include "config.h" // CONFIG_*
2#include "output.h" // dprintf
3#include "string.h" // memcpy_fl
4#include "util.h" // timer_calc
5#include "x86.h" // readl
6#include "malloc.h" // memalign_low
7#include "pci.h" // pci_bdf_to_bus
Kevin O'Connorec443ff2013-12-05 18:43:20 -05008#include "pci_ids.h" // PCI_CLASS_SERIAL_USB_XHCI
Gerd Hoffmanne144bb72013-06-03 16:30:18 +02009#include "pci_regs.h" // PCI_BASE_ADDRESS_0
10#include "usb.h" // struct usb_s
11#include "usb-xhci.h" // struct ehci_qh
12#include "biosvar.h" // GET_LOWFLAT
13
14// --------------------------------------------------------------
15// configuration
16
17#define XHCI_RING_ITEMS 16
18#define XHCI_RING_SIZE (XHCI_RING_ITEMS*sizeof(struct xhci_trb))
19
20/*
21 * xhci_ring structs are allocated with XHCI_RING_SIZE alignment,
22 * then we can get it from a trb pointer (provided by evt ring).
23 */
24#define XHCI_RING(_trb) \
25 ((struct xhci_ring*)((u32)(_trb) & ~(XHCI_RING_SIZE-1)))
26
27// --------------------------------------------------------------
28// bit definitions
29
30#define XHCI_CMD_RS (1<<0)
31#define XHCI_CMD_HCRST (1<<1)
32#define XHCI_CMD_INTE (1<<2)
33#define XHCI_CMD_HSEE (1<<3)
34#define XHCI_CMD_LHCRST (1<<7)
35#define XHCI_CMD_CSS (1<<8)
36#define XHCI_CMD_CRS (1<<9)
37#define XHCI_CMD_EWE (1<<10)
38#define XHCI_CMD_EU3S (1<<11)
39
40#define XHCI_STS_HCH (1<<0)
41#define XHCI_STS_HSE (1<<2)
42#define XHCI_STS_EINT (1<<3)
43#define XHCI_STS_PCD (1<<4)
44#define XHCI_STS_SSS (1<<8)
45#define XHCI_STS_RSS (1<<9)
46#define XHCI_STS_SRE (1<<10)
47#define XHCI_STS_CNR (1<<11)
48#define XHCI_STS_HCE (1<<12)
49
50#define XHCI_PORTSC_CCS (1<<0)
51#define XHCI_PORTSC_PED (1<<1)
52#define XHCI_PORTSC_OCA (1<<3)
53#define XHCI_PORTSC_PR (1<<4)
54#define XHCI_PORTSC_PLS_SHIFT 5
55#define XHCI_PORTSC_PLS_MASK 0xf
56#define XHCI_PORTSC_PP (1<<9)
57#define XHCI_PORTSC_SPEED_SHIFT 10
58#define XHCI_PORTSC_SPEED_MASK 0xf
59#define XHCI_PORTSC_SPEED_FULL (1<<10)
60#define XHCI_PORTSC_SPEED_LOW (2<<10)
61#define XHCI_PORTSC_SPEED_HIGH (3<<10)
62#define XHCI_PORTSC_SPEED_SUPER (4<<10)
63#define XHCI_PORTSC_PIC_SHIFT 14
64#define XHCI_PORTSC_PIC_MASK 0x3
65#define XHCI_PORTSC_LWS (1<<16)
66#define XHCI_PORTSC_CSC (1<<17)
67#define XHCI_PORTSC_PEC (1<<18)
68#define XHCI_PORTSC_WRC (1<<19)
69#define XHCI_PORTSC_OCC (1<<20)
70#define XHCI_PORTSC_PRC (1<<21)
71#define XHCI_PORTSC_PLC (1<<22)
72#define XHCI_PORTSC_CEC (1<<23)
73#define XHCI_PORTSC_CAS (1<<24)
74#define XHCI_PORTSC_WCE (1<<25)
75#define XHCI_PORTSC_WDE (1<<26)
76#define XHCI_PORTSC_WOE (1<<27)
77#define XHCI_PORTSC_DR (1<<30)
78#define XHCI_PORTSC_WPR (1<<31)
79
80#define TRB_C (1<<0)
81#define TRB_TYPE_SHIFT 10
82#define TRB_TYPE_MASK 0x3f
83#define TRB_TYPE(t) (((t) >> TRB_TYPE_SHIFT) & TRB_TYPE_MASK)
84
85#define TRB_EV_ED (1<<2)
86
87#define TRB_TR_ENT (1<<1)
88#define TRB_TR_ISP (1<<2)
89#define TRB_TR_NS (1<<3)
90#define TRB_TR_CH (1<<4)
91#define TRB_TR_IOC (1<<5)
92#define TRB_TR_IDT (1<<6)
93#define TRB_TR_TBC_SHIFT 7
94#define TRB_TR_TBC_MASK 0x3
95#define TRB_TR_BEI (1<<9)
96#define TRB_TR_TLBPC_SHIFT 16
97#define TRB_TR_TLBPC_MASK 0xf
98#define TRB_TR_FRAMEID_SHIFT 20
99#define TRB_TR_FRAMEID_MASK 0x7ff
100#define TRB_TR_SIA (1<<31)
101
102#define TRB_TR_DIR (1<<16)
103
104#define TRB_CR_SLOTID_SHIFT 24
105#define TRB_CR_SLOTID_MASK 0xff
106#define TRB_CR_EPID_SHIFT 16
107#define TRB_CR_EPID_MASK 0x1f
108
109#define TRB_CR_BSR (1<<9)
110#define TRB_CR_DC (1<<9)
111
112#define TRB_LK_TC (1<<1)
113
114#define TRB_INTR_SHIFT 22
115#define TRB_INTR_MASK 0x3ff
116#define TRB_INTR(t) (((t).status >> TRB_INTR_SHIFT) & TRB_INTR_MASK)
117
118typedef enum TRBType {
119 TRB_RESERVED = 0,
120 TR_NORMAL,
121 TR_SETUP,
122 TR_DATA,
123 TR_STATUS,
124 TR_ISOCH,
125 TR_LINK,
126 TR_EVDATA,
127 TR_NOOP,
128 CR_ENABLE_SLOT,
129 CR_DISABLE_SLOT,
130 CR_ADDRESS_DEVICE,
131 CR_CONFIGURE_ENDPOINT,
132 CR_EVALUATE_CONTEXT,
133 CR_RESET_ENDPOINT,
134 CR_STOP_ENDPOINT,
135 CR_SET_TR_DEQUEUE,
136 CR_RESET_DEVICE,
137 CR_FORCE_EVENT,
138 CR_NEGOTIATE_BW,
139 CR_SET_LATENCY_TOLERANCE,
140 CR_GET_PORT_BANDWIDTH,
141 CR_FORCE_HEADER,
142 CR_NOOP,
143 ER_TRANSFER = 32,
144 ER_COMMAND_COMPLETE,
145 ER_PORT_STATUS_CHANGE,
146 ER_BANDWIDTH_REQUEST,
147 ER_DOORBELL,
148 ER_HOST_CONTROLLER,
149 ER_DEVICE_NOTIFICATION,
150 ER_MFINDEX_WRAP,
151} TRBType;
152
153typedef enum TRBCCode {
154 CC_INVALID = 0,
155 CC_SUCCESS,
156 CC_DATA_BUFFER_ERROR,
157 CC_BABBLE_DETECTED,
158 CC_USB_TRANSACTION_ERROR,
159 CC_TRB_ERROR,
160 CC_STALL_ERROR,
161 CC_RESOURCE_ERROR,
162 CC_BANDWIDTH_ERROR,
163 CC_NO_SLOTS_ERROR,
164 CC_INVALID_STREAM_TYPE_ERROR,
165 CC_SLOT_NOT_ENABLED_ERROR,
166 CC_EP_NOT_ENABLED_ERROR,
167 CC_SHORT_PACKET,
168 CC_RING_UNDERRUN,
169 CC_RING_OVERRUN,
170 CC_VF_ER_FULL,
171 CC_PARAMETER_ERROR,
172 CC_BANDWIDTH_OVERRUN,
173 CC_CONTEXT_STATE_ERROR,
174 CC_NO_PING_RESPONSE_ERROR,
175 CC_EVENT_RING_FULL_ERROR,
176 CC_INCOMPATIBLE_DEVICE_ERROR,
177 CC_MISSED_SERVICE_ERROR,
178 CC_COMMAND_RING_STOPPED,
179 CC_COMMAND_ABORTED,
180 CC_STOPPED,
181 CC_STOPPED_LENGTH_INVALID,
182 CC_MAX_EXIT_LATENCY_TOO_LARGE_ERROR = 29,
183 CC_ISOCH_BUFFER_OVERRUN = 31,
184 CC_EVENT_LOST_ERROR,
185 CC_UNDEFINED_ERROR,
186 CC_INVALID_STREAM_ID_ERROR,
187 CC_SECONDARY_BANDWIDTH_ERROR,
188 CC_SPLIT_TRANSACTION_ERROR
189} TRBCCode;
190
191enum {
192 PLS_U0 = 0,
193 PLS_U1 = 1,
194 PLS_U2 = 2,
195 PLS_U3 = 3,
196 PLS_DISABLED = 4,
197 PLS_RX_DETECT = 5,
198 PLS_INACTIVE = 6,
199 PLS_POLLING = 7,
200 PLS_RECOVERY = 8,
201 PLS_HOT_RESET = 9,
202 PLS_COMPILANCE_MODE = 10,
203 PLS_TEST_MODE = 11,
204 PLS_RESUME = 15,
205};
206
207#define xhci_get_field(data, field) \
208 (((data) >> field##_SHIFT) & field##_MASK)
209
210// --------------------------------------------------------------
211// state structs
212
213struct xhci_ring {
214 struct xhci_trb ring[XHCI_RING_ITEMS];
215 struct xhci_trb evt;
216 u32 eidx;
217 u32 nidx;
218 u32 cs;
219 struct mutex_s lock;
220};
221
222struct usb_xhci_s {
223 struct usb_s usb;
224 struct usbhub_s hub;
225
226 /* devinfo */
227 u32 baseaddr;
228 u32 xcap;
229 u32 ports;
230 u32 slots;
231
232 /* xhci registers */
233 struct xhci_caps *caps;
234 struct xhci_op *op;
235 struct xhci_pr *pr;
236 struct xhci_ir *ir;
237 struct xhci_db *db;
238
239 /* xhci data structures */
240 struct xhci_devlist *devs;
241 struct xhci_ring *cmds;
242 struct xhci_ring *evts;
243 struct xhci_er_seg *eseg;
244
245 /* usb devices */
246 struct hlist_head list;
247};
248
249struct xhci_device {
250 struct xhci_devctx devctx;
251 struct xhci_inctx inctx;
252
253 struct usbdevice_s *usbdev;
254 struct usb_xhci_s *xhci;
255 u32 slotid;
256 struct hlist_node next;
257};
258
259struct xhci_pipe {
260 struct xhci_ring reqs;
261
262 struct usb_pipe pipe;
263 struct xhci_device *dev;
264 u32 epid;
265 void *buf;
266 int bufused;
267};
268
269// --------------------------------------------------------------
270// tables
271
272static const char *speed_name[16] = {
273 [ 0 ] = " - ",
274 [ 1 ] = "Full",
275 [ 2 ] = "Low",
276 [ 3 ] = "High",
277 [ 4 ] = "Super",
278};
279
280static const int speed_from_xhci[16] = {
Paul Menzelf08e8472013-10-03 11:55:48 +0200281 [ 0 ] = -1,
Gerd Hoffmanne144bb72013-06-03 16:30:18 +0200282 [ 1 ] = USB_FULLSPEED,
283 [ 2 ] = USB_LOWSPEED,
284 [ 3 ] = USB_HIGHSPEED,
285 [ 4 ] = USB_SUPERSPEED,
Paul Menzelf08e8472013-10-03 11:55:48 +0200286 [ 5 ... 15 ] = -1,
Gerd Hoffmanne144bb72013-06-03 16:30:18 +0200287};
288
289static const int speed_to_xhci[] = {
290 [ USB_FULLSPEED ] = 1,
291 [ USB_LOWSPEED ] = 2,
292 [ USB_HIGHSPEED ] = 3,
293 [ USB_SUPERSPEED ] = 4,
294};
295
296static const int speed_to_ctlsize[] = {
297 [ USB_FULLSPEED ] = 8,
298 [ USB_LOWSPEED ] = 8,
299 [ USB_HIGHSPEED ] = 64,
300 [ USB_SUPERSPEED ] = 256,
301};
302
303static const int eptype_to_xhci_in[] = {
304 [ USB_ENDPOINT_XFER_CONTROL] = 4,
305 [ USB_ENDPOINT_XFER_ISOC ] = 5,
306 [ USB_ENDPOINT_XFER_BULK ] = 6,
307 [ USB_ENDPOINT_XFER_INT ] = 7,
308};
309
310static const int eptype_to_xhci_out[] = {
311 [ USB_ENDPOINT_XFER_CONTROL] = 4,
312 [ USB_ENDPOINT_XFER_ISOC ] = 1,
313 [ USB_ENDPOINT_XFER_BULK ] = 2,
314 [ USB_ENDPOINT_XFER_INT ] = 3,
315};
316
317// --------------------------------------------------------------
318// internal functions, 16bit + 32bit
319
320static void xhci_doorbell(struct usb_xhci_s *xhci, u32 slotid, u32 value)
321{
322 struct xhci_db *db = GET_LOWFLAT(xhci->db);
323 u32 addr = (u32)(&db[slotid].doorbell);
324 pci_writel(addr, value);
325}
326
327static void xhci_process_events(struct usb_xhci_s *xhci)
328{
329 struct xhci_ring *evts = GET_LOWFLAT(xhci->evts);
330
331 for (;;) {
332 /* check for event */
333 u32 nidx = GET_LOWFLAT(evts->nidx);
334 u32 cs = GET_LOWFLAT(evts->cs);
335 struct xhci_trb *etrb = evts->ring + nidx;
336 u32 control = GET_LOWFLAT(etrb->control);
337 if ((control & TRB_C) != (cs ? 1 : 0))
338 return;
339
340 /* process event */
341 u32 evt_type = TRB_TYPE(control);
342 u32 evt_cc = (GET_LOWFLAT(etrb->status) >> 24) & 0xff;
343 switch (evt_type) {
344 case ER_TRANSFER:
345 case ER_COMMAND_COMPLETE:
346 {
347 struct xhci_trb *rtrb = (void*)GET_LOWFLAT(etrb->ptr_low);
348 struct xhci_ring *ring = XHCI_RING(rtrb);
349 struct xhci_trb *evt = &ring->evt;
350 u32 eidx = rtrb - ring->ring + 1;
351 dprintf(5, "%s: ring %p [trb %p, evt %p, type %d, eidx %d, cc %d]\n",
352 __func__, ring, rtrb, evt, evt_type, eidx, evt_cc);
353 memcpy_fl(evt, etrb, sizeof(*etrb));
354 SET_LOWFLAT(ring->eidx, eidx);
355 break;
356 }
357 case ER_PORT_STATUS_CHANGE:
358 {
359 u32 portid = (GET_LOWFLAT(etrb->ptr_low) >> 24) & 0xff;
360 dprintf(3, "%s: status change port #%d\n",
361 __func__, portid);
362 break;
363 }
364 default:
365 dprintf(1, "%s: unknown event, type %d, cc %d\n",
366 __func__, evt_type, evt_cc);
367 break;
368 }
369
370 /* move ring index, notify xhci */
371 nidx++;
372 if (nidx == XHCI_RING_ITEMS) {
373 nidx = 0;
374 cs = cs ? 0 : 1;
375 SET_LOWFLAT(evts->cs, cs);
376 }
377 SET_LOWFLAT(evts->nidx, nidx);
378 struct xhci_ir *ir = GET_LOWFLAT(xhci->ir);
379 u32 addr = (u32)(&ir->erdp_low);
380 u32 erdp = (u32)(evts->ring + nidx);
381 pci_writel(addr, erdp);
382 }
383}
384
385static int xhci_ring_busy(struct xhci_ring *ring)
386{
387 u32 eidx = GET_LOWFLAT(ring->eidx);
388 u32 nidx = GET_LOWFLAT(ring->nidx);
389 return (eidx != nidx);
390}
391
392static int xhci_event_wait(struct usb_xhci_s *xhci,
393 struct xhci_ring *ring,
394 u32 timeout)
395{
396 u32 end = timer_calc(timeout);
397
398 for (;;) {
399 xhci_process_events(xhci);
400 if (!xhci_ring_busy(ring)) {
401 u32 status = GET_LOWFLAT(ring->evt.status);
402 return (status >> 24) & 0xff;
403 }
404 if (timer_check(end)) {
405 warn_timeout();
406 return -1;
407 }
408 yield();
409 }
410}
411
412static void xhci_trb_queue(struct xhci_ring *ring,
413 struct xhci_trb *trb)
414{
415 u32 nidx = GET_LOWFLAT(ring->nidx);
416 u32 cs = GET_LOWFLAT(ring->cs);
417 struct xhci_trb *dst;
418 u32 control;
419
420 if (nidx == XHCI_RING_ITEMS-1) {
421 dst = ring->ring + nidx;
422 control = (TR_LINK << 10); // trb type
423 control |= TRB_LK_TC;
424 control |= (cs ? TRB_C : 0);
425 SET_LOWFLAT(dst->ptr_low, (u32)&ring[0]);
426 SET_LOWFLAT(dst->ptr_high, 0);
427 SET_LOWFLAT(dst->status, 0);
428 SET_LOWFLAT(dst->control, control);
429 nidx = 0;
430 cs = cs ? 0 : 1;
431 SET_LOWFLAT(ring->nidx, nidx);
432 SET_LOWFLAT(ring->cs, cs);
433
434 dprintf(5, "%s: ring %p [linked]\n", __func__, ring);
435 }
436
437 dst = ring->ring + nidx;
438 control = GET_LOWFLAT(trb->control) | (cs ? TRB_C : 0);
439
440 SET_LOWFLAT(dst->ptr_low, GET_LOWFLAT(trb->ptr_low));
441 SET_LOWFLAT(dst->ptr_high, GET_LOWFLAT(trb->ptr_high));
442 SET_LOWFLAT(dst->status, GET_LOWFLAT(trb->status));
443 SET_LOWFLAT(dst->control, control);
444 nidx++;
445 SET_LOWFLAT(ring->nidx, nidx);
446
447 dprintf(5, "%s: ring %p [nidx %d, len %d]\n",
448 __func__, ring, nidx,
449 GET_LOWFLAT(trb->status) & 0xffff);
450}
451
452static void xhci_xfer_queue(struct xhci_pipe *pipe,
453 struct xhci_trb *trb)
454{
455 xhci_trb_queue(&pipe->reqs, trb);
456}
457
458static void xhci_xfer_kick(struct xhci_pipe *pipe)
459{
460 struct xhci_device *dev = GET_LOWFLAT(pipe->dev);
461 struct usb_xhci_s *xhci = GET_LOWFLAT(dev->xhci);
462 u32 slotid = GET_LOWFLAT(dev->slotid);
463 u32 epid = GET_LOWFLAT(pipe->epid);
464
465 dprintf(5, "%s: ring %p, slotid %d, epid %d\n",
466 __func__, &pipe->reqs, slotid, epid);
467 xhci_doorbell(xhci, slotid, epid);
468}
469
470static void xhci_xfer_normal(struct xhci_pipe *pipe,
471 void *data, int datalen)
472{
473 struct xhci_trb trb;
474
475 memset(&trb, 0, sizeof(trb));
476 trb.ptr_low = (u32)data;
477 trb.status = datalen;
478 trb.control |= (TR_NORMAL << 10); // trb type
479 trb.control |= TRB_TR_IOC;
480
481 xhci_xfer_queue(pipe, MAKE_FLATPTR(GET_SEG(SS), &trb));
482 xhci_xfer_kick(pipe);
483}
484
485// --------------------------------------------------------------
486// internal functions, pure 32bit
487
488static int wait_bit(u32 *reg, u32 mask, int value, u32 timeout)
489{
490 ASSERT32FLAT();
491 u32 end = timer_calc(timeout);
492
493 while ((readl(reg) & mask) != value) {
494 if (timer_check(end)) {
495 warn_timeout();
496 return -1;
497 }
498 yield();
499 }
500 return 0;
501}
502
503static int xhci_cmd_submit(struct usb_xhci_s *xhci,
504 struct xhci_trb *cmd)
505{
506 ASSERT32FLAT();
507 int rc;
508
509 mutex_lock(&xhci->cmds->lock);
510 xhci_trb_queue(xhci->cmds, cmd);
511 xhci_doorbell(xhci, 0, 0);
512 rc = xhci_event_wait(xhci, xhci->cmds, 1000);
513 mutex_unlock(&xhci->cmds->lock);
514 return rc;
515}
516
517static int xhci_cmd_enable_slot(struct usb_xhci_s *xhci)
518{
519 ASSERT32FLAT();
520 struct xhci_trb cmd = {
521 .ptr_low = 0,
522 .ptr_high = 0,
523 .status = 0,
524 .control = (CR_ENABLE_SLOT << 10)
525 };
526 dprintf(3, "%s:\n", __func__);
527 int cc = xhci_cmd_submit(xhci, &cmd);
528 if (cc != CC_SUCCESS)
529 return -1;
530 return (xhci->cmds->evt.control >> 24) & 0xff;
531}
532
533static int xhci_cmd_disable_slot(struct xhci_device *dev)
534{
535 ASSERT32FLAT();
536 struct xhci_trb cmd = {
537 .ptr_low = 0,
538 .ptr_high = 0,
539 .status = 0,
540 .control = (dev->slotid << 24) | (CR_DISABLE_SLOT << 10)
541 };
542 dprintf(3, "%s: slotid %d\n", __func__, dev->slotid);
543 return xhci_cmd_submit(dev->xhci, &cmd);
544}
545
546static int xhci_cmd_address_device(struct xhci_device *dev)
547{
548 ASSERT32FLAT();
549 struct xhci_trb cmd = {
550 .ptr_low = (u32)&dev->inctx,
551 .ptr_high = 0,
552 .status = 0,
553 .control = (dev->slotid << 24) | (CR_ADDRESS_DEVICE << 10)
554 };
555 dprintf(3, "%s: slotid %d\n", __func__, dev->slotid);
556 return xhci_cmd_submit(dev->xhci, &cmd);
557}
558
559static int xhci_cmd_configure_endpoint(struct xhci_device *dev)
560{
561 ASSERT32FLAT();
562 struct xhci_trb cmd = {
563 .ptr_low = (u32)&dev->inctx,
564 .ptr_high = 0,
565 .status = 0,
566 .control = (dev->slotid << 24) | (CR_CONFIGURE_ENDPOINT << 10)
567 };
568 dprintf(3, "%s: slotid %d, add 0x%x, del 0x%x\n", __func__,
569 dev->slotid, dev->inctx.add, dev->inctx.del);
570 return xhci_cmd_submit(dev->xhci, &cmd);
571}
572
573static int xhci_cmd_evaluate_context(struct xhci_device *dev)
574{
575 ASSERT32FLAT();
576 struct xhci_trb cmd = {
577 .ptr_low = (u32)&dev->inctx,
578 .ptr_high = 0,
579 .status = 0,
580 .control = (dev->slotid << 24) | (CR_EVALUATE_CONTEXT << 10)
581 };
582 dprintf(3, "%s: slotid %d, add 0x%x, del 0x%x\n", __func__,
583 dev->slotid, dev->inctx.add, dev->inctx.del);
584 return xhci_cmd_submit(dev->xhci, &cmd);
585}
586
587static void xhci_xfer_setup(struct xhci_pipe *pipe,
588 const struct usb_ctrlrequest *req,
589 int dir, int datalen)
590{
591 ASSERT32FLAT();
592 struct xhci_trb trb;
593
594 memset(&trb, 0, sizeof(trb));
595 trb.ptr_low |= req->bRequestType;
596 trb.ptr_low |= (req->bRequest) << 8;
597 trb.ptr_low |= (req->wValue) << 16;
598 trb.ptr_high |= req->wIndex;
599 trb.ptr_high |= (req->wLength) << 16;
600 trb.status |= 8; // length
601 trb.control |= (TR_SETUP << 10); // trb type
602 trb.control |= TRB_TR_IDT;
603 if (datalen)
604 trb.control |= (dir ? 3 : 2) << 16; // transfer type
605 xhci_xfer_queue(pipe, &trb);
606}
607
608static void xhci_xfer_data(struct xhci_pipe *pipe,
609 int dir, void *data, int datalen)
610{
611 ASSERT32FLAT();
612 struct xhci_trb trb;
613
614 memset(&trb, 0, sizeof(trb));
615 trb.ptr_low = (u32)data;
616 trb.status = datalen;
617 trb.control |= (TR_DATA << 10); // trb type
618 if (dir)
619 trb.control |= (1 << 16);
620 xhci_xfer_queue(pipe, &trb);
621}
622
623static void xhci_xfer_status(struct xhci_pipe *pipe, int dir)
624{
625 ASSERT32FLAT();
626 struct xhci_trb trb;
627
628 memset(&trb, 0, sizeof(trb));
629 trb.control |= (TR_STATUS << 10); // trb type
630 trb.control |= TRB_TR_IOC;
631 if (dir)
632 trb.control |= (1 << 16);
633
634 xhci_xfer_queue(pipe, &trb);
635 xhci_xfer_kick(pipe);
636}
637
638static struct xhci_device *xhci_find_alloc_device(struct usb_xhci_s *xhci,
639 struct usbdevice_s *usbdev)
640{
641 ASSERT32FLAT();
642 struct xhci_device *dev;
643
644 hlist_for_each_entry(dev, &xhci->list, next) {
645 if (dev->usbdev == usbdev) {
646 return dev;
647 }
648 }
649
650 dev = memalign_low(64, sizeof(*dev));
651 if (!dev) {
652 warn_noalloc();
653 return NULL;
654 }
655 memset(dev, 0, sizeof(*dev));
656 dev->usbdev = usbdev;
657 dev->xhci = xhci;
658 hlist_add_head(&dev->next, &xhci->list);
659 return dev;
660}
661
662static void
663configure_xhci(void *data)
664{
665 ASSERT32FLAT();
666 struct usb_xhci_s *xhci = data;
667 u32 reg;
668
669 xhci->devs = memalign_high(64, sizeof(*xhci->devs) * (xhci->slots + 1));
670 xhci->eseg = memalign_high(64, sizeof(*xhci->eseg));
671 xhci->cmds = memalign_high(XHCI_RING_SIZE, sizeof(*xhci->cmds));
672 xhci->evts = memalign_low(XHCI_RING_SIZE, sizeof(*xhci->evts));
673 if (!xhci->devs || !xhci->cmds || !xhci->evts || !xhci->eseg) {
674 warn_noalloc();
675 goto fail;
676 }
677 memset(xhci->devs, 0, sizeof(*xhci->devs) * (xhci->slots + 1));
678 memset(xhci->cmds, 0, sizeof(*xhci->cmds));
679 memset(xhci->evts, 0, sizeof(*xhci->evts));
680 memset(xhci->eseg, 0, sizeof(*xhci->eseg));
681
682 reg = readl(&xhci->op->usbcmd);
683 if (reg & XHCI_CMD_RS) {
684 reg &= ~XHCI_CMD_RS;
685 writel(&xhci->op->usbcmd, reg);
686 if (wait_bit(&xhci->op->usbsts, XHCI_STS_HCH, XHCI_STS_HCH, 32) != 0)
687 goto fail;
688 }
689
690 dprintf(3, "%s: resetting\n", __func__);
691 writel(&xhci->op->usbcmd, XHCI_CMD_HCRST);
692 if (wait_bit(&xhci->op->usbcmd, XHCI_CMD_HCRST, 0, 100) != 0)
693 goto fail;
694 if (wait_bit(&xhci->op->usbsts, XHCI_STS_CNR, 0, 100) != 0)
695 goto fail;
696
697 writel(&xhci->op->config, xhci->slots);
698 writel(&xhci->op->dcbaap_low, (u32)xhci->devs);
699 writel(&xhci->op->dcbaap_high, 0);
700 writel(&xhci->op->crcr_low, (u32)xhci->cmds | 1);
701 writel(&xhci->op->crcr_high, 0);
702 xhci->cmds->cs = 1;
703
704 xhci->eseg->ptr_low = (u32)xhci->evts;
705 xhci->eseg->ptr_high = 0;
706 xhci->eseg->size = XHCI_RING_ITEMS;
707 writel(&xhci->ir->erstsz, 1);
708 writel(&xhci->ir->erdp_low, (u32)xhci->evts);
709 writel(&xhci->ir->erdp_high, 0);
710 writel(&xhci->ir->erstba_low, (u32)xhci->eseg);
711 writel(&xhci->ir->erstba_high, 0);
712 xhci->evts->cs = 1;
713
714 reg = readl(&xhci->op->usbcmd);
715 reg |= XHCI_CMD_RS;
716 writel(&xhci->op->usbcmd, reg);
717
718 // FIXME: try find a more elegant way than a fixed delay
719 mdelay(100);
720
721 usb_enumerate(&xhci->hub);
722 if (xhci->hub.devcount)
723 return;
724
725 // No devices found - shutdown and free controller.
726 dprintf(1, "XHCI no devices found\n");
727 reg = readl(&xhci->op->usbcmd);
728 reg &= ~XHCI_CMD_RS;
729 writel(&xhci->op->usbcmd, reg);
730 wait_bit(&xhci->op->usbsts, XHCI_STS_HCH, XHCI_STS_HCH, 32);
731
732fail:
733 free(xhci->eseg);
734 free(xhci->evts);
735 free(xhci->cmds);
736 free(xhci->devs);
737 free(xhci);
738}
739
740// --------------------------------------------------------------
741// xhci root hub
742
743// Check if device attached to port
744static void
745xhci_print_port_state(int loglevel, const char *prefix, u32 port, u32 portsc)
746{
747 ASSERT32FLAT();
748 u32 pls = xhci_get_field(portsc, XHCI_PORTSC_PLS);
749 u32 speed = xhci_get_field(portsc, XHCI_PORTSC_SPEED);
750
751 dprintf(loglevel, "%s port #%d: 0x%08x,%s%s pls %d, speed %d [%s]\n",
752 prefix, port + 1, portsc,
753 (portsc & XHCI_PORTSC_PP) ? " powered," : "",
754 (portsc & XHCI_PORTSC_PED) ? " enabled," : "",
755 pls, speed, speed_name[speed]);
756}
757
758static int
759xhci_hub_detect(struct usbhub_s *hub, u32 port)
760{
761 ASSERT32FLAT();
762 struct usb_xhci_s *xhci = container_of(hub->cntl, struct usb_xhci_s, usb);
763 u32 portsc = readl(&xhci->pr[port].portsc);
764
765 xhci_print_port_state(3, __func__, port, portsc);
766 switch (xhci_get_field(portsc, XHCI_PORTSC_PLS)) {
767 case PLS_U0:
768 case PLS_POLLING:
769 return 0;
770 default:
771 return -1;
772 }
773}
774
775// Reset device on port
776static int
777xhci_hub_reset(struct usbhub_s *hub, u32 port)
778{
779 ASSERT32FLAT();
780 struct usb_xhci_s *xhci = container_of(hub->cntl, struct usb_xhci_s, usb);
781 u32 portsc = readl(&xhci->pr[port].portsc);
782 int rc;
783
784 switch (xhci_get_field(portsc, XHCI_PORTSC_PLS)) {
785 case PLS_U0:
786 rc = speed_from_xhci[xhci_get_field(portsc, XHCI_PORTSC_SPEED)];
787 break;
788 case PLS_POLLING:
789 xhci_print_port_state(3, __func__, port, portsc);
790 portsc |= XHCI_PORTSC_PR;
791 writel(&xhci->pr[port].portsc, portsc);
792 if (wait_bit(&xhci->pr[port].portsc, XHCI_PORTSC_PED, XHCI_PORTSC_PED, 100) != 0)
793 return -1;
794 portsc = readl(&xhci->pr[port].portsc);
795 rc = speed_from_xhci[xhci_get_field(portsc, XHCI_PORTSC_SPEED)];
796 break;
797 default:
798 rc = -1;
799 break;
800 }
801
802 xhci_print_port_state(1, "XHCI", port, portsc);
803 return rc;
804}
805
806static void
807xhci_hub_disconnect(struct usbhub_s *hub, u32 port)
808{
809 ASSERT32FLAT();
810 struct usb_xhci_s *xhci = container_of(hub->cntl, struct usb_xhci_s, usb);
811 struct xhci_device *dev;
812
813 hlist_for_each_entry(dev, &xhci->list, next) {
814 if (dev->usbdev->hub == hub &&
815 dev->usbdev->port == port &&
816 dev->slotid != 0) {
817 xhci_cmd_disable_slot(dev);
818 hlist_del(&dev->next);
819 return;
820 }
821 }
822}
823
824static struct usbhub_op_s xhci_hub_ops = {
825 .detect = xhci_hub_detect,
826 .reset = xhci_hub_reset,
827 .disconnect = xhci_hub_disconnect,
828};
829
830// --------------------------------------------------------------
831// external interface
832
833struct usb_pipe *
834xhci_alloc_pipe(struct usbdevice_s *usbdev
835 , struct usb_endpoint_descriptor *epdesc)
836{
837 ASSERT32FLAT();
838 if (!CONFIG_USB_XHCI)
839 return NULL;
840 u8 eptype = epdesc->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK;
841 struct usb_xhci_s *xhci = container_of(
842 usbdev->hub->cntl, struct usb_xhci_s, usb);
843 struct xhci_pipe *pipe;
844 u32 epid;
845
846 if (epdesc->bEndpointAddress == 0) {
847 epid = 1;
848 } else {
849 epid = (epdesc->bEndpointAddress & 0x0f) * 2;
850 epid += (epdesc->bEndpointAddress & USB_DIR_IN) ? 1 : 0;
851 }
852
853 if (eptype == USB_ENDPOINT_XFER_CONTROL)
854 pipe = memalign_high(XHCI_RING_SIZE, sizeof(*pipe));
855 else
856 pipe = memalign_low(XHCI_RING_SIZE, sizeof(*pipe));
857 if (!pipe) {
858 warn_noalloc();
859 return NULL;
860 }
861 memset(pipe, 0, sizeof(*pipe));
862
863 usb_desc2pipe(&pipe->pipe, usbdev, epdesc);
864 pipe->dev = xhci_find_alloc_device(xhci, usbdev);
865 if (!pipe->dev) {
866 free(pipe);
867 return NULL;
868 }
869 pipe->epid = epid;
870 pipe->reqs.cs = 1;
871 if (eptype == USB_ENDPOINT_XFER_INT)
872 pipe->buf = malloc_low(pipe->pipe.maxpacket);
873
874 dprintf(3, "%s: usbdev %p, ring %p, slotid %d, epid %d\n", __func__,
875 usbdev, &pipe->reqs, pipe->dev->slotid, pipe->epid);
876 if (pipe->epid > 1 && pipe->dev->slotid) {
877 struct xhci_inctx *in = &pipe->dev->inctx;
878 in->add = (1 << pipe->epid) | 1;
879 in->del = 0;
880
881 in->slot.ctx[0] |= (31 << 27); // context entries
882
883 int e = pipe->epid-1;
884 in->ep[e].ctx[1] |= (eptype << 3);
885 if (epdesc->bEndpointAddress & USB_DIR_IN)
886 in->ep[e].ctx[1] |= (1 << 5);
887 in->ep[e].ctx[1] |= (pipe->pipe.maxpacket << 16);
888 in->ep[e].deq_low = (u32)&pipe->reqs.ring[0];
889 in->ep[e].deq_low |= 1; // dcs
890 in->ep[e].deq_high = 0;
891 in->ep[e].length = pipe->pipe.maxpacket;
892
893 int cc = xhci_cmd_configure_endpoint(pipe->dev);
894 if (cc != CC_SUCCESS) {
895 dprintf(1, "%s: configure endpoint: failed (cc %d)\n", __func__, cc);
896 free(pipe);
897 return NULL;
898 }
899 }
900
901 return &pipe->pipe;
902}
903
904struct usb_pipe *
905xhci_update_pipe(struct usbdevice_s *usbdev, struct usb_pipe *upipe
906 , struct usb_endpoint_descriptor *epdesc)
907{
908 ASSERT32FLAT();
909 if (!CONFIG_USB_XHCI)
910 return NULL;
911 u8 eptype = epdesc->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK;
912 struct xhci_pipe *pipe = container_of(upipe, struct xhci_pipe, pipe);
913 dprintf(3, "%s: usbdev %p, ring %p, slotid %d, epid %d\n", __func__,
914 usbdev, &pipe->reqs, pipe->dev->slotid, pipe->epid);
915 if (eptype == USB_ENDPOINT_XFER_CONTROL &&
916 pipe->pipe.maxpacket != epdesc->wMaxPacketSize) {
917 dprintf(1, "%s: reconf ctl endpoint pkt size: %d -> %d\n",
918 __func__, pipe->pipe.maxpacket, epdesc->wMaxPacketSize);
919 pipe->pipe.maxpacket = epdesc->wMaxPacketSize;
920 struct xhci_inctx *in = &pipe->dev->inctx;
921 in->add = (1 << 1);
922 in->del = 0;
923 in->ep[0].ctx[1] &= 0xffff;
924 in->ep[0].ctx[1] |= (pipe->pipe.maxpacket << 16);
925 int cc = xhci_cmd_evaluate_context(pipe->dev);
926 if (cc != CC_SUCCESS) {
927 dprintf(1, "%s: reconf ctl endpoint: failed (cc %d)\n",
928 __func__, cc);
929 }
930 }
931 return upipe;
932}
933
934int
935xhci_control(struct usb_pipe *p, int dir, const void *cmd, int cmdsize
936 , void *data, int datalen)
937{
938 ASSERT32FLAT();
939 if (!CONFIG_USB_XHCI)
940 return -1;
941 const struct usb_ctrlrequest *req = cmd;
942 struct xhci_pipe *pipe = container_of(p, struct xhci_pipe, pipe);
943 struct usb_xhci_s *xhci = pipe->dev->xhci;
944 int cc;
945
946 if (req->bRequest == USB_REQ_SET_ADDRESS) {
947 int slotid = xhci_cmd_enable_slot(xhci);
948 if (slotid < 0) {
949 dprintf(1, "%s: enable slot: failed\n", __func__);
950 return -1;
951 }
952 dprintf(3, "%s: enable slot: got slotid %d\n", __func__, slotid);
953 pipe->dev->slotid = slotid;
954 xhci->devs[slotid].ptr_low = (u32)&pipe->dev->devctx;
955 xhci->devs[slotid].ptr_high = 0;
956
957 struct usbdevice_s *usbdev = pipe->dev->usbdev;
958 u32 route = 0;
959 while (usbdev->hub->usbdev) {
960 route <<= 4;
961 route |= (usbdev->port+1) & 0xf;
962 usbdev = usbdev->hub->usbdev;
963 }
964 dprintf(3, "%s: root port %d, route 0x%x\n",
965 __func__, usbdev->port+1, route);
966
967 struct xhci_inctx *in = &pipe->dev->inctx;
968 in->add = 0x03;
969 in->slot.ctx[0] |= (1 << 27); // context entries
970 in->slot.ctx[0] |= speed_to_xhci[pipe->dev->usbdev->speed] << 20;
971 in->slot.ctx[0] |= route;
972 in->slot.ctx[1] |= (usbdev->port+1) << 16;
973 /* TODO ctx0: hub bit */
974 /* TODO ctx1: hub ports */
975
976 in->ep[0].ctx[0] |= (3 << 16); // interval: 1ms
977 in->ep[0].ctx[1] |= (4 << 3); // control pipe
978 in->ep[0].ctx[1] |= (speed_to_ctlsize[pipe->dev->usbdev->speed] << 16);
979
980 in->ep[0].deq_low = (u32)&pipe->reqs.ring[0];
981 in->ep[0].deq_low |= 1; // dcs
982 in->ep[0].deq_high = 0;
983 in->ep[0].length = 8;
984
985 cc = xhci_cmd_address_device(pipe->dev);
986 if (cc != CC_SUCCESS) {
987 dprintf(1, "%s: address device: failed (cc %d)\n", __func__, cc);
988 return -1;
989 }
990 return 0;
991 }
992
993 xhci_xfer_setup(pipe, req, dir, datalen);
994 if (datalen)
995 xhci_xfer_data(pipe, dir, data, datalen);
996 xhci_xfer_status(pipe, dir);
997
998 cc = xhci_event_wait(xhci, &pipe->reqs, 1000);
999 if (cc != CC_SUCCESS) {
1000 dprintf(1, "%s: control xfer failed (cc %d)\n", __func__, cc);
1001 return -1;
1002 }
1003
1004 return 0;
1005}
1006
1007int
1008xhci_send_bulk(struct usb_pipe *p, int dir, void *data, int datalen)
1009{
1010 if (!CONFIG_USB_XHCI)
1011 return -1;
1012
1013 struct xhci_pipe *pipe = container_of(p, struct xhci_pipe, pipe);
1014 struct xhci_device *dev = GET_LOWFLAT(pipe->dev);
1015 struct usb_xhci_s *xhci = GET_LOWFLAT(dev->xhci);
1016
1017 xhci_xfer_normal(pipe, data, datalen);
1018 int cc = xhci_event_wait(xhci, &pipe->reqs, 1000);
1019 if (cc != CC_SUCCESS) {
1020 dprintf(1, "%s: bulk xfer failed (cc %d)\n", __func__, cc);
1021 return -1;
1022 }
1023 return 0;
1024}
1025
1026int
1027xhci_poll_intr(struct usb_pipe *p, void *data)
1028{
1029 if (!CONFIG_USB_XHCI)
1030 return -1;
1031
1032 struct xhci_pipe *pipe = container_of(p, struct xhci_pipe, pipe);
1033 struct xhci_device *dev = GET_LOWFLAT(pipe->dev);
1034 struct usb_xhci_s *xhci = GET_LOWFLAT(dev->xhci);
1035 u32 len = GET_LOWFLAT(pipe->pipe.maxpacket);
1036 void *buf = GET_LOWFLAT(pipe->buf);
1037 int bufused = GET_LOWFLAT(pipe->bufused);
1038
1039 if (!bufused) {
1040 xhci_xfer_normal(pipe, buf, len);
1041 bufused = 1;
1042 SET_LOWFLAT(pipe->bufused, bufused);
1043 return -1;
1044 }
1045
1046 xhci_process_events(xhci);
1047 if (xhci_ring_busy(&pipe->reqs))
1048 return -1;
1049 dprintf(5, "%s: st %x ct %x [ %p <= %p / %d ]\n", __func__,
1050 GET_LOWFLAT(pipe->reqs.evt.status),
1051 GET_LOWFLAT(pipe->reqs.evt.control),
1052 MAKE_FLATPTR(GET_SEG(SS), data), buf, len);
1053 memcpy_fl(MAKE_FLATPTR(GET_SEG(SS), data), buf, len);
1054 xhci_xfer_normal(pipe, buf, len);
1055 return 0;
1056}
1057
Kevin O'Connorec443ff2013-12-05 18:43:20 -05001058static void
1059xhci_controller_setup(struct pci_device *pci)
Gerd Hoffmanne144bb72013-06-03 16:30:18 +02001060{
Gerd Hoffmanne144bb72013-06-03 16:30:18 +02001061 struct usb_xhci_s *xhci = malloc_low(sizeof(*xhci));
1062 if (!xhci) {
1063 warn_noalloc();
Kevin O'Connorec443ff2013-12-05 18:43:20 -05001064 return;
Gerd Hoffmanne144bb72013-06-03 16:30:18 +02001065 }
1066 memset(xhci, 0, sizeof(*xhci));
1067
Kevin O'Connorec443ff2013-12-05 18:43:20 -05001068 wait_preempt(); // Avoid pci_config_readl when preempting
Gerd Hoffmanne144bb72013-06-03 16:30:18 +02001069 xhci->baseaddr = pci_config_readl(pci->bdf, PCI_BASE_ADDRESS_0)
1070 & PCI_BASE_ADDRESS_MEM_MASK;
1071 xhci->caps = (void*)(xhci->baseaddr);
1072 xhci->op = (void*)(xhci->baseaddr + readb(&xhci->caps->caplength));
1073 xhci->pr = (void*)(xhci->baseaddr + readb(&xhci->caps->caplength) + 0x400);
1074 xhci->db = (void*)(xhci->baseaddr + readl(&xhci->caps->dboff));
1075 xhci->ir = (void*)(xhci->baseaddr + readl(&xhci->caps->rtsoff) + 0x20);
1076
1077 u32 hcs1 = readl(&xhci->caps->hcsparams1);
1078 u32 hcc = readl(&xhci->caps->hccparams);
1079 xhci->ports = (hcs1 >> 24) & 0xff;
1080 xhci->slots = hcs1 & 0xff;
1081 xhci->xcap = ((hcc >> 16) & 0xffff) << 2;
1082
Gerd Hoffmanne144bb72013-06-03 16:30:18 +02001083 xhci->usb.pci = pci;
1084 xhci->usb.type = USB_TYPE_XHCI;
1085 xhci->hub.cntl = &xhci->usb;
1086 xhci->hub.portcount = xhci->ports;
1087 xhci->hub.op = &xhci_hub_ops;
1088
1089 dprintf(1, "XHCI init on dev %02x:%02x.%x: regs @ %p, %d ports, %d slots\n"
1090 , pci_bdf_to_bus(pci->bdf), pci_bdf_to_dev(pci->bdf)
1091 , pci_bdf_to_fn(pci->bdf), xhci->caps
1092 , xhci->ports, xhci->slots);
1093
1094 if (xhci->xcap) {
1095 u32 off, addr = xhci->baseaddr + xhci->xcap;
1096 do {
1097 struct xhci_xcap *xcap = (void*)addr;
1098 u32 ports, name, cap = readl(&xcap->cap);
1099 switch (cap & 0xff) {
1100 case 0x02:
1101 name = readl(&xcap->data[0]);
1102 ports = readl(&xcap->data[1]);
1103 dprintf(1, "XHCI protocol %c%c%c%c %x.%02x, %d ports (offset %d)\n"
1104 , (name >> 0) & 0xff
1105 , (name >> 8) & 0xff
1106 , (name >> 16) & 0xff
1107 , (name >> 24) & 0xff
1108 , (cap >> 24) & 0xff
1109 , (cap >> 16) & 0xff
1110 , (ports >> 8) & 0xff
1111 , (ports >> 0) & 0xff);
1112 break;
1113 default:
1114 dprintf(1, "XHCI extcap 0x%x @ %x\n", cap & 0xff, addr);
1115 break;
1116 }
1117 off = (cap >> 8) & 0xff;
1118 addr += off << 2;
1119 } while (off > 0);
1120 }
1121
1122 pci_config_maskw(pci->bdf, PCI_COMMAND, 0, PCI_COMMAND_MASTER);
1123
1124 run_thread(configure_xhci, xhci);
Kevin O'Connorec443ff2013-12-05 18:43:20 -05001125}
1126
1127void
1128xhci_setup(void)
1129{
1130 if (! CONFIG_USB_XHCI)
1131 return;
1132 struct pci_device *pci;
1133 foreachpci(pci) {
1134 if (pci_classprog(pci) == PCI_CLASS_SERIAL_USB_XHCI)
1135 xhci_controller_setup(pci);
1136 }
Gerd Hoffmanne144bb72013-06-03 16:30:18 +02001137}