blob: 21a16d27abe0f3f8e96833dd4a70700e27021802 [file] [log] [blame]
Julius Werner12de6982014-01-13 11:13:23 -08001/*
2 * Optimized assembly for low-level CPU operations on ARMv7 processors.
3 *
4 * Cache flushing code based off sys/arch/arm/arm/cpufunc_asm_armv7.S in NetBSD
5 *
6 * Copyright (c) 2010 Per Odlund <per.odlund@armagedon.se>
7 * Copyright (c) 2014 Google Inc.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * 3. The name of the author may not be used to endorse or promote products
18 * derived from this software without specific prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 */
32
Julius Werner1f8d2462014-01-15 14:13:25 -080033#include <arch/asm.h>
Hakim Giydan43e5b572016-09-08 10:13:59 -070034#include <rules.h>
Julius Werner1f8d2462014-01-15 14:13:25 -080035
Julius Werner12de6982014-01-13 11:13:23 -080036/*
Julius Werner1f8d2462014-01-15 14:13:25 -080037 * Dcache invalidations by set/way work by passing a [way:sbz:set:sbz:level:0]
38 * bitfield in a register to the appropriate MCR instruction. This algorithm
39 * works by initializing a bitfield with the highest-numbered set and way, and
40 * generating a "set decrement" and a "way decrement". The former just contains
41 * the LSB of the set field, but the latter contains the LSB of the way field
42 * minus the highest valid set field... such that when you subtract it from a
43 * [way:0:level] field you end up with a [way - 1:highest_set:level] field
44 * through the magic of double subtraction. It's quite ingenius, really.
45 * Takes care to only use r0-r3 and ip so it's pefectly ABI-compatible without
46 * needing to write to memory.
Vadim Bendeburybe7124e2015-03-02 11:33:13 -080047 *
48 * THIS FUNCTION MUST PRESERVE THE VALUE OF r10
Julius Werner12de6982014-01-13 11:13:23 -080049 */
50
Julius Werner1f8d2462014-01-15 14:13:25 -080051.macro dcache_apply_all crm
52 dsb
53 mov r3, #-2 @ initialize level so that we start at 0
Julius Werner12de6982014-01-13 11:13:23 -080054
Julius Werner1f8d2462014-01-15 14:13:25 -0800551: @next_level
56 add r3, r3, #2 @ increment level
57
58 mrc p15, 1, r0, c0, c0, 1 @ read CLIDR
59 and ip, r0, #0x07000000 @ narrow to LoC
60 lsr ip, ip, #23 @ left align LoC (low 4 bits)
61 cmp r3, ip @ compare
62 bge 3f @done @ else fall through (r0 == CLIDR)
63
64 add r2, r3, r3, lsr #1 @ r2 = (level << 1) * 3 / 2
Julius Werner12de6982014-01-13 11:13:23 -080065 mov r1, r0, lsr r2 @ r1 = cache type
Julius Werner638015b2014-04-04 14:01:15 -070066 and r1, r1, #7
Julius Werner12de6982014-01-13 11:13:23 -080067 cmp r1, #2 @ is it data or i&d?
Julius Werner1f8d2462014-01-15 14:13:25 -080068 blt 1b @next_level @ nope, skip level
Julius Werner12de6982014-01-13 11:13:23 -080069
70 mcr p15, 2, r3, c0, c0, 0 @ select cache level
71 isb
72 mrc p15, 1, r0, c0, c0, 0 @ read CCSIDR
73
74 ubfx ip, r0, #0, #3 @ get linesize from CCSIDR
75 add ip, ip, #4 @ apply bias
76 ubfx r2, r0, #13, #15 @ get numsets - 1 from CCSIDR
77 lsl r2, r2, ip @ shift to set position
78 orr r3, r3, r2 @ merge set into way/set/level
79 mov r1, #1
80 lsl r1, r1, ip @ r1 = set decr
81
82 ubfx ip, r0, #3, #10 @ get numways - 1 from [to be discarded] CCSIDR
83 clz r2, ip @ number of bits to MSB of way
84 lsl ip, ip, r2 @ shift by that into way position
Julius Werner1f8d2462014-01-15 14:13:25 -080085 mov r0, #1
Julius Werner12de6982014-01-13 11:13:23 -080086 lsl r2, r0, r2 @ r2 now contains the way decr
87 mov r0, r3 @ get sets/level (no way yet)
88 orr r3, r3, ip @ merge way into way/set/level
89 bfc r0, #0, #4 @ clear low 4 bits (level) to get numset - 1
90 sub r2, r2, r0 @ subtract from way decr
91
92 /* r3 = ways/sets/level, r2 = way decr, r1 = set decr, r0 and ip are free */
Julius Werner1f8d2462014-01-15 14:13:25 -0800932: mcr p15, 0, r3, c7, \crm, 2 @ writeback and/or invalidate line
Julius Werner12de6982014-01-13 11:13:23 -080094 cmp r3, #15 @ are we done with this level (way/set == 0)
Julius Werner1f8d2462014-01-15 14:13:25 -080095 bls 1b @next_level @ yes, go to next level
96 lsr r0, r3, #4 @ clear level bits leaving only way/set bits
97 lsls r0, r0, #14 @ clear way bits leaving only set bits
Julius Werner12de6982014-01-13 11:13:23 -080098 subne r3, r3, r1 @ non-zero?, decrement set #
99 subeq r3, r3, r2 @ zero?, decrement way # and restore set count
Julius Werner1f8d2462014-01-15 14:13:25 -0800100 b 2b
Julius Werner12de6982014-01-13 11:13:23 -0800101
Julius Werner1f8d2462014-01-15 14:13:25 -08001023: @done
Julius Werner12de6982014-01-13 11:13:23 -0800103 mov r0, #0 @ default back to cache level 0
104 mcr p15, 2, r0, c0, c0, 0 @ select cache level
105 dsb
106 isb
107 bx lr
Julius Werner1f8d2462014-01-15 14:13:25 -0800108.endm
109
Julius Wernerfd9defc2014-01-21 20:11:22 -0800110/*
111 * Bring an ARM processor we just gained control of (e.g. from IROM) into a
112 * known state regarding caches/SCTLR. Completely cleans and invalidates
113 * icache/dcache, disables MMU and dcache (if active), and enables unaligned
114 * accesses, icache and branch prediction (if inactive). Clobbers r4 and r5.
Vadim Bendeburybe7124e2015-03-02 11:33:13 -0800115 *
116 * THIS FUNCTION MUST PRESERVE THE VALUE OF r10
Julius Wernerfd9defc2014-01-21 20:11:22 -0800117 */
118ENTRY(arm_init_caches)
119 /* r4: SCTLR, return address: r5 (stay valid for the whole function) */
120 mov r5, lr
121 mrc p15, 0, r4, c1, c0, 0
122
123 /* Activate ICache (12) and Branch Prediction (11) already for speed */
124 orr r4, # (1 << 11) | (1 << 12)
125 mcr p15, 0, r4, c1, c0, 0
126
127 /* Flush and invalidate dcache in ascending order */
Daisuke Nojiri4e158bc2015-02-06 19:13:34 -0800128 bl dcache_invalidate_all
Julius Wernerfd9defc2014-01-21 20:11:22 -0800129
Hakim Giydan43e5b572016-09-08 10:13:59 -0700130#if ENV_ARMV7_A
Julius Wernerfd9defc2014-01-21 20:11:22 -0800131 /* Deactivate MMU (0), Alignment Check (1) and DCache (2) */
132 and r4, # ~(1 << 0) & ~(1 << 1) & ~(1 << 2)
133 mcr p15, 0, r4, c1, c0, 0
134
135 /* Invalidate icache and TLB for good measure */
136 mcr p15, 0, r0, c7, c5, 0
137 mcr p15, 0, r0, c8, c7, 0
Hakim Giydan43e5b572016-09-08 10:13:59 -0700138#endif
139
140#if ENV_ARMV7_R
141 /* Deactivate Alignment Check (1) and DCache (2) */
142 and r4, # ~(1 << 1) & ~(1 << 2)
143 mcr p15, 0, r4, c1, c0, 0
144
145 /* Invalidate icache for good measure */
146 mcr p15, 0, r0, c7, c5, 0
147#endif
Julius Wernerfd9defc2014-01-21 20:11:22 -0800148 dsb
149 isb
150
151 bx r5
152ENDPROC(arm_init_caches)
153
Julius Werner1f8d2462014-01-15 14:13:25 -0800154ENTRY(dcache_invalidate_all)
155 dcache_apply_all crm=c6
156ENDPROC(dcache_invalidate_all)
157
158ENTRY(dcache_clean_all)
159 dcache_apply_all crm=c10
160ENDPROC(dcache_clean_all)
161
162ENTRY(dcache_clean_invalidate_all)
163 dcache_apply_all crm=c14
164ENDPROC(dcache_clean_invalidate_all)