David Hendricks | bba8090 | 2013-03-14 15:24:57 -0700 | [diff] [blame] | 1 | /* |
| 2 | * This file is part of the coreboot project. |
| 3 | * |
| 4 | * Copyright 2013 Google Inc. |
| 5 | * |
| 6 | * Redistribution and use in source and binary forms, with or without |
| 7 | * modification, are permitted provided that the following conditions |
| 8 | * are met: |
| 9 | * 1. Redistributions of source code must retain the above copyright |
| 10 | * notice, this list of conditions and the following disclaimer. |
| 11 | * 2. Redistributions in binary form must reproduce the above copyright |
| 12 | * notice, this list of conditions and the following disclaimer in the |
| 13 | * documentation and/or other materials provided with the distribution. |
| 14 | * 3. The name of the author may not be used to endorse or promote products |
| 15 | * derived from this software without specific prior written permission. |
| 16 | * |
| 17 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND |
| 18 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
| 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
| 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE |
| 21 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
| 22 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS |
| 23 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
| 24 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
| 25 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
| 26 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
| 27 | * SUCH DAMAGE. |
| 28 | * |
David Hendricks | a54efdc | 2013-03-19 17:32:54 -0700 | [diff] [blame] | 29 | * cache.c: Cache maintenance routines for ARMv7-A and ARMv7-R |
| 30 | * |
| 31 | * Reference: ARM Architecture Reference Manual, ARMv7-A and ARMv7-R edition |
David Hendricks | bba8090 | 2013-03-14 15:24:57 -0700 | [diff] [blame] | 32 | */ |
| 33 | |
David Hendricks | fa244a6 | 2013-03-28 18:07:30 -0700 | [diff] [blame] | 34 | #include <stdint.h> |
David Hendricks | bba8090 | 2013-03-14 15:24:57 -0700 | [diff] [blame] | 35 | |
| 36 | #include <arch/cache.h> |
| 37 | |
| 38 | #define bitmask(high, low) ((1UL << (high)) + \ |
| 39 | ((1UL << (high)) - 1) - ((1UL << (low)) - 1)) |
| 40 | |
| 41 | /* Basic log2() implementation. Note: log2(0) is 0 for our purposes. */ |
| 42 | /* FIXME: src/include/lib.h is difficult to work with due to romcc */ |
| 43 | static unsigned long log2(unsigned long u) |
| 44 | { |
| 45 | int i = 0; |
| 46 | |
| 47 | while (u >>= 1) |
| 48 | i++; |
| 49 | |
| 50 | return i; |
| 51 | } |
| 52 | |
| 53 | void tlb_invalidate_all(void) |
| 54 | { |
| 55 | /* |
| 56 | * FIXME: ARMv7 Architecture Ref. Manual claims that the distinction |
David Hendricks | a54efdc | 2013-03-19 17:32:54 -0700 | [diff] [blame] | 57 | * instruction vs. data TLBs is deprecated in ARMv7, however this does |
| 58 | * not seem to be the case as of Cortex-A15. |
David Hendricks | bba8090 | 2013-03-14 15:24:57 -0700 | [diff] [blame] | 59 | */ |
| 60 | tlbiall(); |
| 61 | dtlbiall(); |
| 62 | itlbiall(); |
| 63 | isb(); |
| 64 | dsb(); |
| 65 | } |
| 66 | |
| 67 | void icache_invalidate_all(void) |
| 68 | { |
David Hendricks | a54efdc | 2013-03-19 17:32:54 -0700 | [diff] [blame] | 69 | /* |
| 70 | * icache can be entirely invalidated with one operation. |
David Hendricks | bba8090 | 2013-03-14 15:24:57 -0700 | [diff] [blame] | 71 | * Note: If branch predictors are architecturally-visible, ICIALLU |
| 72 | * also performs a BPIALL operation (B2-1283 in arch manual) |
| 73 | */ |
| 74 | iciallu(); |
| 75 | isb(); |
| 76 | } |
| 77 | |
| 78 | enum dcache_op { |
David Hendricks | b98ab4a | 2013-08-16 12:17:50 -0700 | [diff] [blame] | 79 | OP_DCCSW, |
David Hendricks | bba8090 | 2013-03-14 15:24:57 -0700 | [diff] [blame] | 80 | OP_DCCISW, |
David Hendricks | 758abdd | 2013-03-19 17:57:59 -0700 | [diff] [blame] | 81 | OP_DCISW, |
| 82 | OP_DCCIMVAC, |
David Hendricks | 426ce41 | 2013-03-19 18:38:48 -0700 | [diff] [blame] | 83 | OP_DCCMVAC, |
David Hendricks | b98ab4a | 2013-08-16 12:17:50 -0700 | [diff] [blame] | 84 | OP_DCIMVAC, |
David Hendricks | bba8090 | 2013-03-14 15:24:57 -0700 | [diff] [blame] | 85 | }; |
| 86 | |
David Hendricks | a54efdc | 2013-03-19 17:32:54 -0700 | [diff] [blame] | 87 | /* |
| 88 | * Do a dcache operation on entire cache by set/way. This is done for |
| 89 | * portability because mapping of memory address to cache location is |
| 90 | * implementation defined (See note on "Requirements for operations by |
| 91 | * set/way" in arch ref. manual). |
| 92 | */ |
David Hendricks | bba8090 | 2013-03-14 15:24:57 -0700 | [diff] [blame] | 93 | static void dcache_op_set_way(enum dcache_op op) |
| 94 | { |
| 95 | uint32_t ccsidr; |
| 96 | unsigned int associativity, num_sets, linesize_bytes; |
| 97 | unsigned int set, way; |
| 98 | unsigned int level; |
| 99 | |
| 100 | level = (read_csselr() >> 1) & 0x7; |
| 101 | |
| 102 | /* |
| 103 | * dcache must be invalidated by set/way for portability since virtual |
| 104 | * memory mapping is system-defined. The number of sets and |
| 105 | * associativity is given by CCSIDR. We'll use DCISW to invalidate the |
| 106 | * dcache. |
| 107 | */ |
| 108 | ccsidr = read_ccsidr(); |
| 109 | |
| 110 | /* FIXME: rounding up required here? */ |
| 111 | num_sets = ((ccsidr & bitmask(27, 13)) >> 13) + 1; |
| 112 | associativity = ((ccsidr & bitmask(12, 3)) >> 3) + 1; |
| 113 | /* FIXME: do we need to use CTR.DminLine here? */ |
| 114 | linesize_bytes = (1 << ((ccsidr & 0x7) + 2)) * 4; |
| 115 | |
David Hendricks | e85f4eb | 2013-03-26 21:34:01 -0700 | [diff] [blame] | 116 | dsb(); |
| 117 | |
David Hendricks | bba8090 | 2013-03-14 15:24:57 -0700 | [diff] [blame] | 118 | /* |
| 119 | * Set/way operations require an interesting bit packing. See section |
| 120 | * B4-35 in the ARMv7 Architecture Reference Manual: |
| 121 | * |
| 122 | * A: Log2(associativity) |
| 123 | * B: L+S |
| 124 | * L: Log2(linesize) |
| 125 | * S: Log2(num_sets) |
| 126 | * |
| 127 | * The bits are packed as follows: |
| 128 | * 31 31-A B B-1 L L-1 4 3 1 0 |
| 129 | * |---|-------------|--------|-------|-----|-| |
| 130 | * |Way| zeros | Set | zeros |level|0| |
| 131 | * |---|-------------|--------|-------|-----|-| |
| 132 | */ |
| 133 | for (way = 0; way < associativity; way++) { |
| 134 | for (set = 0; set < num_sets; set++) { |
| 135 | uint32_t val = 0; |
| 136 | val |= way << (32 - log2(associativity)); |
| 137 | val |= set << log2(linesize_bytes); |
| 138 | val |= level << 1; |
| 139 | switch(op) { |
| 140 | case OP_DCCISW: |
| 141 | dccisw(val); |
| 142 | break; |
| 143 | case OP_DCISW: |
| 144 | dcisw(val); |
| 145 | break; |
David Hendricks | b98ab4a | 2013-08-16 12:17:50 -0700 | [diff] [blame] | 146 | case OP_DCCSW: |
| 147 | dccsw(val); |
| 148 | break; |
David Hendricks | bba8090 | 2013-03-14 15:24:57 -0700 | [diff] [blame] | 149 | default: |
| 150 | break; |
| 151 | } |
| 152 | } |
| 153 | } |
David Hendricks | e85f4eb | 2013-03-26 21:34:01 -0700 | [diff] [blame] | 154 | isb(); |
David Hendricks | bba8090 | 2013-03-14 15:24:57 -0700 | [diff] [blame] | 155 | } |
| 156 | |
David Hendricks | 7b19f66 | 2013-03-28 18:26:03 -0700 | [diff] [blame] | 157 | static void dcache_foreach(enum dcache_op op) |
| 158 | { |
| 159 | uint32_t clidr; |
| 160 | int level; |
| 161 | |
| 162 | clidr = read_clidr(); |
| 163 | for (level = 0; level < 7; level++) { |
| 164 | unsigned int ctype = (clidr >> (level * 3)) & 0x7; |
| 165 | uint32_t csselr; |
| 166 | |
| 167 | switch(ctype) { |
| 168 | case 0x2: |
| 169 | case 0x3: |
| 170 | case 0x4: |
| 171 | csselr = level << 1; |
| 172 | write_csselr(csselr); |
| 173 | dcache_op_set_way(op); |
| 174 | break; |
| 175 | default: |
| 176 | /* no cache, icache only, or reserved */ |
| 177 | break; |
| 178 | } |
| 179 | } |
| 180 | } |
| 181 | |
David Hendricks | b98ab4a | 2013-08-16 12:17:50 -0700 | [diff] [blame] | 182 | void dcache_clean_all(void) |
| 183 | { |
| 184 | dcache_foreach(OP_DCCSW); |
| 185 | } |
| 186 | |
David Hendricks | bba8090 | 2013-03-14 15:24:57 -0700 | [diff] [blame] | 187 | void dcache_clean_invalidate_all(void) |
| 188 | { |
David Hendricks | 7b19f66 | 2013-03-28 18:26:03 -0700 | [diff] [blame] | 189 | dcache_foreach(OP_DCCISW); |
David Hendricks | bba8090 | 2013-03-14 15:24:57 -0700 | [diff] [blame] | 190 | } |
| 191 | |
| 192 | void dcache_invalidate_all(void) |
| 193 | { |
David Hendricks | 7b19f66 | 2013-03-28 18:26:03 -0700 | [diff] [blame] | 194 | dcache_foreach(OP_DCISW); |
David Hendricks | bba8090 | 2013-03-14 15:24:57 -0700 | [diff] [blame] | 195 | } |
| 196 | |
Gabe Black | d40be11 | 2013-10-09 23:45:07 -0700 | [diff] [blame] | 197 | unsigned int dcache_line_bytes(void) |
David Hendricks | bba8090 | 2013-03-14 15:24:57 -0700 | [diff] [blame] | 198 | { |
| 199 | uint32_t ccsidr; |
Gabe Black | d40be11 | 2013-10-09 23:45:07 -0700 | [diff] [blame] | 200 | static unsigned int line_bytes = 0; |
| 201 | |
| 202 | if (line_bytes) |
| 203 | return line_bytes; |
David Hendricks | bba8090 | 2013-03-14 15:24:57 -0700 | [diff] [blame] | 204 | |
| 205 | ccsidr = read_ccsidr(); |
| 206 | /* [2:0] - Indicates (Log2(number of words in cache line)) - 2 */ |
Gabe Black | d40be11 | 2013-10-09 23:45:07 -0700 | [diff] [blame] | 207 | line_bytes = 1 << ((ccsidr & 0x7) + 2); /* words per line */ |
| 208 | line_bytes *= sizeof(unsigned int); /* bytes per line */ |
David Hendricks | bba8090 | 2013-03-14 15:24:57 -0700 | [diff] [blame] | 209 | |
Gabe Black | d40be11 | 2013-10-09 23:45:07 -0700 | [diff] [blame] | 210 | return line_bytes; |
David Hendricks | bba8090 | 2013-03-14 15:24:57 -0700 | [diff] [blame] | 211 | } |
| 212 | |
David Hendricks | 758abdd | 2013-03-19 17:57:59 -0700 | [diff] [blame] | 213 | /* |
| 214 | * Do a dcache operation by modified virtual address. This is useful for |
| 215 | * maintaining coherency in drivers which do DMA transfers and only need to |
| 216 | * perform cache maintenance on a particular memory range rather than the |
| 217 | * entire cache. |
| 218 | */ |
Julius Werner | f09f224 | 2013-08-28 14:43:14 -0700 | [diff] [blame] | 219 | static void dcache_op_mva(void const *addr, size_t len, enum dcache_op op) |
David Hendricks | bba8090 | 2013-03-14 15:24:57 -0700 | [diff] [blame] | 220 | { |
David Hendricks | 19f3092 | 2013-03-26 17:47:05 -0700 | [diff] [blame] | 221 | unsigned long line, linesize; |
David Hendricks | bba8090 | 2013-03-14 15:24:57 -0700 | [diff] [blame] | 222 | |
Gabe Black | d40be11 | 2013-10-09 23:45:07 -0700 | [diff] [blame] | 223 | linesize = dcache_line_bytes(); |
Julius Werner | f09f224 | 2013-08-28 14:43:14 -0700 | [diff] [blame] | 224 | line = (uint32_t)addr & ~(linesize - 1); |
David Hendricks | 42f5513 | 2013-03-25 19:50:11 -0700 | [diff] [blame] | 225 | |
| 226 | dsb(); |
Julius Werner | f09f224 | 2013-08-28 14:43:14 -0700 | [diff] [blame] | 227 | while ((void *)line < addr + len) { |
David Hendricks | 758abdd | 2013-03-19 17:57:59 -0700 | [diff] [blame] | 228 | switch(op) { |
| 229 | case OP_DCCIMVAC: |
David Hendricks | 19f3092 | 2013-03-26 17:47:05 -0700 | [diff] [blame] | 230 | dccimvac(line); |
David Hendricks | 758abdd | 2013-03-19 17:57:59 -0700 | [diff] [blame] | 231 | break; |
Hung-Te Lin | 86148a6 | 2013-07-08 12:17:25 +0800 | [diff] [blame] | 232 | case OP_DCCMVAC: |
| 233 | dccmvac(line); |
| 234 | break; |
David Hendricks | b98ab4a | 2013-08-16 12:17:50 -0700 | [diff] [blame] | 235 | case OP_DCIMVAC: |
| 236 | dcimvac(line); |
| 237 | break; |
David Hendricks | 758abdd | 2013-03-19 17:57:59 -0700 | [diff] [blame] | 238 | default: |
| 239 | break; |
| 240 | } |
David Hendricks | 19f3092 | 2013-03-26 17:47:05 -0700 | [diff] [blame] | 241 | line += linesize; |
David Hendricks | 758abdd | 2013-03-19 17:57:59 -0700 | [diff] [blame] | 242 | } |
David Hendricks | 42f5513 | 2013-03-25 19:50:11 -0700 | [diff] [blame] | 243 | isb(); |
David Hendricks | 758abdd | 2013-03-19 17:57:59 -0700 | [diff] [blame] | 244 | } |
| 245 | |
Julius Werner | f09f224 | 2013-08-28 14:43:14 -0700 | [diff] [blame] | 246 | void dcache_clean_by_mva(void const *addr, size_t len) |
David Hendricks | 426ce41 | 2013-03-19 18:38:48 -0700 | [diff] [blame] | 247 | { |
| 248 | dcache_op_mva(addr, len, OP_DCCMVAC); |
| 249 | } |
| 250 | |
Julius Werner | f09f224 | 2013-08-28 14:43:14 -0700 | [diff] [blame] | 251 | void dcache_clean_invalidate_by_mva(void const *addr, size_t len) |
David Hendricks | 758abdd | 2013-03-19 17:57:59 -0700 | [diff] [blame] | 252 | { |
| 253 | dcache_op_mva(addr, len, OP_DCCIMVAC); |
David Hendricks | bba8090 | 2013-03-14 15:24:57 -0700 | [diff] [blame] | 254 | } |
| 255 | |
Julius Werner | f09f224 | 2013-08-28 14:43:14 -0700 | [diff] [blame] | 256 | void dcache_invalidate_by_mva(void const *addr, size_t len) |
David Hendricks | b98ab4a | 2013-08-16 12:17:50 -0700 | [diff] [blame] | 257 | { |
| 258 | dcache_op_mva(addr, len, OP_DCIMVAC); |
| 259 | } |
| 260 | |
David Hendricks | f9be756 | 2013-03-21 21:58:50 -0700 | [diff] [blame] | 261 | void dcache_mmu_disable(void) |
| 262 | { |
David Hendricks | 5877935 | 2013-03-29 13:40:09 -0700 | [diff] [blame] | 263 | uint32_t sctlr; |
David Hendricks | dbc11e2 | 2013-03-26 21:39:03 -0700 | [diff] [blame] | 264 | |
David Hendricks | 5877935 | 2013-03-29 13:40:09 -0700 | [diff] [blame] | 265 | dcache_clean_invalidate_all(); |
David Hendricks | f9be756 | 2013-03-21 21:58:50 -0700 | [diff] [blame] | 266 | sctlr = read_sctlr(); |
David Hendricks | f9be756 | 2013-03-21 21:58:50 -0700 | [diff] [blame] | 267 | sctlr &= ~(SCTLR_C | SCTLR_M); |
| 268 | write_sctlr(sctlr); |
| 269 | } |
| 270 | |
| 271 | |
| 272 | void dcache_mmu_enable(void) |
| 273 | { |
| 274 | uint32_t sctlr; |
| 275 | |
| 276 | sctlr = read_sctlr(); |
| 277 | dcache_clean_invalidate_all(); |
| 278 | sctlr |= SCTLR_C | SCTLR_M; |
| 279 | write_sctlr(sctlr); |
| 280 | } |
| 281 | |
Gabe Black | 51edd54 | 2013-09-30 23:00:33 -0700 | [diff] [blame] | 282 | void arm_invalidate_caches(void) |
David Hendricks | 8ec6905 | 2013-03-19 17:11:31 -0700 | [diff] [blame] | 283 | { |
| 284 | uint32_t clidr; |
| 285 | int level; |
| 286 | |
| 287 | /* Invalidate branch predictor */ |
| 288 | bpiall(); |
| 289 | |
| 290 | /* Iterate thru each cache identified in CLIDR and invalidate */ |
| 291 | clidr = read_clidr(); |
| 292 | for (level = 0; level < 7; level++) { |
| 293 | unsigned int ctype = (clidr >> (level * 3)) & 0x7; |
| 294 | uint32_t csselr; |
| 295 | |
| 296 | switch(ctype) { |
| 297 | case 0x0: |
| 298 | /* no cache */ |
| 299 | break; |
| 300 | case 0x1: |
| 301 | /* icache only */ |
| 302 | csselr = (level << 1) | 1; |
| 303 | write_csselr(csselr); |
| 304 | icache_invalidate_all(); |
| 305 | break; |
| 306 | case 0x2: |
| 307 | case 0x4: |
| 308 | /* dcache only or unified cache */ |
David Hendricks | 7762091 | 2013-03-28 18:37:29 -0700 | [diff] [blame] | 309 | csselr = level << 1; |
| 310 | write_csselr(csselr); |
David Hendricks | 8ec6905 | 2013-03-19 17:11:31 -0700 | [diff] [blame] | 311 | dcache_invalidate_all(); |
| 312 | break; |
| 313 | case 0x3: |
| 314 | /* separate icache and dcache */ |
| 315 | csselr = (level << 1) | 1; |
| 316 | write_csselr(csselr); |
| 317 | icache_invalidate_all(); |
| 318 | |
David Hendricks | 8f39887 | 2013-03-28 13:45:19 -0700 | [diff] [blame] | 319 | csselr = level << 1; |
David Hendricks | 8ec6905 | 2013-03-19 17:11:31 -0700 | [diff] [blame] | 320 | write_csselr(csselr); |
| 321 | dcache_invalidate_all(); |
| 322 | break; |
| 323 | default: |
| 324 | /* reserved */ |
| 325 | break; |
| 326 | } |
| 327 | } |
| 328 | |
| 329 | /* Invalidate TLB */ |
| 330 | tlb_invalidate_all(); |
| 331 | } |