David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 1 | /***********************license start*********************************** |
| 2 | * Copyright (c) 2003-2017 Cavium Inc. (support@cavium.com). All rights |
| 3 | * reserved. |
| 4 | * |
| 5 | * |
| 6 | * Redistribution and use in source and binary forms, with or without |
| 7 | * modification, are permitted provided that the following conditions are |
| 8 | * met: |
| 9 | * |
| 10 | * * Redistributions of source code must retain the above copyright |
| 11 | * notice, this list of conditions and the following disclaimer. |
| 12 | * |
| 13 | * * Redistributions in binary form must reproduce the above |
| 14 | * copyright notice, this list of conditions and the following |
| 15 | * disclaimer in the documentation and/or other materials provided |
| 16 | * with the distribution. |
| 17 | * |
| 18 | * * Neither the name of Cavium Inc. nor the names of |
| 19 | * its contributors may be used to endorse or promote products |
| 20 | * derived from this software without specific prior written |
| 21 | * permission. |
| 22 | * |
| 23 | * This Software, including technical data, may be subject to U.S. export |
| 24 | * control laws, including the U.S. Export Administration Act and its |
| 25 | * associated regulations, and may be subject to export or import |
| 26 | * regulations in other countries. |
| 27 | * |
| 28 | * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" |
| 29 | * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR |
| 30 | * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT |
| 31 | * TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY |
| 32 | * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT |
| 33 | * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES |
| 34 | * OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR |
| 35 | * PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, |
| 36 | * QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK |
| 37 | * ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU. |
| 38 | ***********************license end**************************************/ |
| 39 | #include <bdk.h> |
Joel Kitching | a1b1517 | 2020-03-12 18:15:34 +0800 | [diff] [blame] | 40 | #include <string.h> |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 41 | #include "libbdk-arch/bdk-csrs-gti.h" |
| 42 | #include "libbdk-arch/bdk-csrs-ocx.h" |
| 43 | |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 44 | #include <bdk-minimal.h> /* for printf --> printk */ |
| 45 | #include <libbdk-dram/bdk-dram-test.h> |
| 46 | #include <libbdk-hal/bdk-atomic.h> |
| 47 | #include <libbdk-hal/bdk-clock.h> |
| 48 | #include <libbdk-hal/bdk-utils.h> |
| 49 | #include <libbdk-os/bdk-init.h> |
| 50 | #include <libbdk-os/bdk-thread.h> |
Elyes Haouas | bdd03c2 | 2024-05-27 11:20:07 +0200 | [diff] [blame^] | 51 | #include <stdio.h> |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 52 | |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 53 | /* This code is an optional part of the BDK. It is only linked in |
| 54 | if BDK_REQUIRE() needs it */ |
| 55 | BDK_REQUIRE_DEFINE(DRAM_TEST); |
| 56 | |
| 57 | #define MAX_ERRORS_TO_REPORT 50 |
| 58 | #define RETRY_LIMIT 1000 |
| 59 | |
| 60 | typedef struct |
| 61 | { |
| 62 | const char * name; /* Friendly name for the test */ |
| 63 | __bdk_dram_test_t test_func; /* Function to call */ |
| 64 | int bursts; /* Bursts parameter to pass to the test */ |
| 65 | int max_cores; /* Maximum number of cores the test should be run on in parallel. Zero means all */ |
| 66 | } dram_test_info_t; |
| 67 | |
| 68 | static const dram_test_info_t TEST_INFO[] = { |
| 69 | /* Name, Test function, Bursts, Max Cores */ |
| 70 | { "Data Bus", __bdk_dram_test_mem_data_bus, 8, 1}, |
| 71 | { "Address Bus", __bdk_dram_test_mem_address_bus, 0, 1}, |
| 72 | { "Marching Rows", __bdk_dram_test_mem_rows, 16, 0}, |
| 73 | { "Random Data", __bdk_dram_test_mem_random, 32, 0}, |
| 74 | { "Random XOR (32 Burst)", __bdk_dram_test_mem_xor, 32, 0}, |
| 75 | { "Self Address", __bdk_dram_test_mem_self_addr, 1, 0}, |
| 76 | { "March C- Solid Bits", __bdk_dram_test_mem_solid, 1, 0}, |
| 77 | { "March C- Checkerboard", __bdk_dram_test_mem_checkerboard, 1, 0}, |
| 78 | { "Walking Ones Left", __bdk_dram_test_mem_leftwalk1, 1, 0}, |
| 79 | { "Walking Ones Right", __bdk_dram_test_mem_rightwalk1, 1, 0}, |
| 80 | { "Walking Zeros Left", __bdk_dram_test_mem_leftwalk0, 1, 0}, |
| 81 | { "Walking Zeros Right", __bdk_dram_test_mem_rightwalk0, 1, 0}, |
| 82 | { "Random XOR (224 Burst)", __bdk_dram_test_mem_xor, 224, 0}, |
| 83 | { "Fast Scan", __bdk_dram_test_fast_scan, 0, 0}, |
| 84 | { NULL, NULL, 0, 0} |
| 85 | }; |
| 86 | |
| 87 | /* These variables count the number of ECC errors. They should only be accessed atomically */ |
| 88 | int64_t __bdk_dram_ecc_single_bit_errors[BDK_MAX_MEM_CHANS]; |
| 89 | int64_t __bdk_dram_ecc_double_bit_errors[BDK_MAX_MEM_CHANS]; |
| 90 | |
| 91 | static int64_t dram_test_thread_done; |
| 92 | static int64_t dram_test_thread_errors; |
| 93 | static uint64_t dram_test_thread_start; |
| 94 | static uint64_t dram_test_thread_end; |
| 95 | static uint64_t dram_test_thread_size; |
| 96 | |
| 97 | /** |
| 98 | * Force the memory at the pointer location to be written to memory and evicted |
| 99 | * from L2. L1 will be unaffected. |
| 100 | * |
| 101 | * @param address Physical memory location |
| 102 | */ |
| 103 | void __bdk_dram_flush_to_mem(uint64_t address) |
| 104 | { |
| 105 | BDK_MB; |
Jacob Garber | 4926e98 | 2019-07-26 11:45:43 -0600 | [diff] [blame] | 106 | char *ptr = bdk_phys_to_ptr(address); |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 107 | BDK_CACHE_WBI_L2(ptr); |
| 108 | } |
| 109 | |
| 110 | /** |
| 111 | * Force a memory region to be written to DRAM and evicted from L2 |
| 112 | * |
| 113 | * @param area Start of the region |
| 114 | * @param max_address |
| 115 | * End of the region (exclusive) |
| 116 | */ |
| 117 | void __bdk_dram_flush_to_mem_range(uint64_t area, uint64_t max_address) |
| 118 | { |
Jacob Garber | 4926e98 | 2019-07-26 11:45:43 -0600 | [diff] [blame] | 119 | char *ptr = bdk_phys_to_ptr(area); |
| 120 | char *end = bdk_phys_to_ptr(max_address); |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 121 | BDK_MB; |
| 122 | while (ptr < end) |
| 123 | { |
| 124 | BDK_CACHE_WBI_L2(ptr); |
| 125 | ptr += 128; |
| 126 | } |
| 127 | } |
| 128 | |
| 129 | /** |
| 130 | * Convert a test enumeration into a string |
| 131 | * |
| 132 | * @param test Test to convert |
| 133 | * |
| 134 | * @return String for display |
| 135 | */ |
| 136 | const char *bdk_dram_get_test_name(int test) |
| 137 | { |
Patrick Georgi | 6b688f5 | 2021-02-12 13:49:11 +0100 | [diff] [blame] | 138 | if (test < (int) ARRAY_SIZE(TEST_INFO)) |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 139 | return TEST_INFO[test].name; |
| 140 | else |
| 141 | return NULL; |
| 142 | } |
| 143 | |
| 144 | static bdk_dram_test_flags_t dram_test_flags; // FIXME: Don't use global |
| 145 | /** |
| 146 | * This function is run as a thread to perform memory tests over multiple cores. |
| 147 | * Each thread gets a section of memory to work on, which is controlled by global |
| 148 | * variables at the beginning of this file. |
| 149 | * |
| 150 | * @param arg Number of the region we should check |
Jonathan Neuschäfer | 45e6c82 | 2018-12-11 17:53:07 +0100 | [diff] [blame] | 151 | * @param arg1 Pointer to the test_info structure |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 152 | */ |
| 153 | static void dram_test_thread(int arg, void *arg1) |
| 154 | { |
| 155 | const dram_test_info_t *test_info = arg1; |
| 156 | const int bursts = test_info->bursts; |
| 157 | const int range_number = arg; |
| 158 | |
| 159 | /* Figure out our work memory range. |
| 160 | * |
| 161 | * Note start_address and end_address just provide the physical offset |
| 162 | * portion of the address and do not have the node bits set. This is |
| 163 | * to simplify address checks and calculations. Later, when about to run |
| 164 | * the memory test, the routines adds in the node bits to form the final |
| 165 | * addresses. |
| 166 | */ |
| 167 | uint64_t start_address = dram_test_thread_start + dram_test_thread_size * range_number; |
| 168 | uint64_t end_address = start_address + dram_test_thread_size; |
| 169 | if (end_address > dram_test_thread_end) |
| 170 | end_address = dram_test_thread_end; |
| 171 | |
| 172 | bdk_node_t test_node = bdk_numa_local(); |
| 173 | if (dram_test_flags & BDK_DRAM_TEST_USE_CCPI) |
| 174 | test_node ^= 1; |
| 175 | /* Insert the node part of the address */ |
| 176 | start_address = bdk_numa_get_address(test_node, start_address); |
| 177 | end_address = bdk_numa_get_address(test_node, end_address); |
| 178 | /* Test the region */ |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 179 | BDK_TRACE(DRAM_TEST, " Node %d, core %d, Testing [0x%011llx:0x%011llx]\n", |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 180 | bdk_numa_local(), bdk_get_core_num() & 127, start_address, end_address - 1); |
| 181 | test_info->test_func(start_address, end_address, bursts); |
| 182 | |
| 183 | /* Report that we're done */ |
| 184 | BDK_TRACE(DRAM_TEST, "Thread %d on node %d done with memory test\n", range_number, bdk_numa_local()); |
| 185 | bdk_atomic_add64_nosync(&dram_test_thread_done, 1); |
| 186 | } |
| 187 | |
| 188 | /** |
| 189 | * Run the memory test. |
| 190 | * |
| 191 | * @param test_info |
| 192 | * @param start_address |
| 193 | * Physical address to start at |
| 194 | * @param length Length of memory block |
| 195 | * @param flags Flags to control memory test options. Zero defaults to testing all |
| 196 | * node with statistics and progress output. |
| 197 | * |
| 198 | * @return Number of errors found. Zero is success. Negative means the test |
| 199 | * did not run due to some other failure. |
| 200 | */ |
| 201 | static int __bdk_dram_run_test(const dram_test_info_t *test_info, uint64_t start_address, |
| 202 | uint64_t length, bdk_dram_test_flags_t flags) |
| 203 | { |
| 204 | /* Figure out the addess of the byte one off the top of memory */ |
| 205 | uint64_t max_address = bdk_dram_get_size_mbytes(bdk_numa_local()); |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 206 | BDK_TRACE(DRAM_TEST, "DRAM available per node: %llu MB\n", max_address); |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 207 | max_address <<= 20; |
| 208 | |
| 209 | /* Make sure we have enough */ |
| 210 | if (max_address < (16<<20)) |
| 211 | { |
| 212 | bdk_error("DRAM size is too small\n"); |
| 213 | return -1; |
| 214 | } |
| 215 | |
| 216 | /* Make sure the amount is sane */ |
| 217 | if (CAVIUM_IS_MODEL(CAVIUM_CN8XXX)) |
| 218 | { |
| 219 | if (max_address > (1ull << 40)) /* 40 bits in CN8XXX */ |
| 220 | max_address = 1ull << 40; |
| 221 | } |
| 222 | else |
| 223 | { |
| 224 | if (max_address > (1ull << 43)) /* 43 bits in CN9XXX */ |
| 225 | max_address = 1ull << 43; |
| 226 | } |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 227 | BDK_TRACE(DRAM_TEST, "DRAM max address: 0x%011llx\n", max_address-1); |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 228 | |
| 229 | /* Make sure the start address is lower than the top of memory */ |
| 230 | if (start_address >= max_address) |
| 231 | { |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 232 | bdk_error("Start address is larger than the amount of memory: 0x%011llx versus 0x%011llx\n", |
| 233 | start_address, max_address); |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 234 | return -1; |
| 235 | } |
| 236 | if (length == (uint64_t)-1) |
| 237 | length = max_address - start_address; |
| 238 | |
| 239 | /* Final range checks */ |
| 240 | uint64_t end_address = start_address + length; |
| 241 | if (end_address > max_address) |
| 242 | { |
| 243 | end_address = max_address; |
| 244 | length = end_address - start_address; |
| 245 | } |
| 246 | if (length == 0) |
| 247 | return 0; |
| 248 | |
| 249 | /* Ready to run the test. Figure out how many cores we need */ |
| 250 | int max_cores = test_info->max_cores; |
| 251 | int total_cores_all_nodes = max_cores; |
| 252 | |
| 253 | /* Figure out the number of cores available in the system */ |
| 254 | if (max_cores == 0) |
| 255 | { |
| 256 | max_cores += bdk_get_num_running_cores(bdk_numa_local()); |
| 257 | /* Calculate the total number of cores being used. The per node number |
| 258 | is confusing to people */ |
| 259 | for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) |
| 260 | if (flags & (1 << node)) |
| 261 | { |
| 262 | if (flags & BDK_DRAM_TEST_USE_CCPI) |
| 263 | total_cores_all_nodes += bdk_get_num_running_cores(node ^ 1); |
| 264 | else |
| 265 | total_cores_all_nodes += bdk_get_num_running_cores(node); |
| 266 | } |
| 267 | } |
| 268 | if (!(flags & BDK_DRAM_TEST_NO_BANNERS)) |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 269 | printf("Starting Test \"%s\" for [0x%011llx:0x%011llx] using %d core(s)\n", |
| 270 | test_info->name, start_address, end_address - 1, total_cores_all_nodes); |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 271 | |
| 272 | /* Remember the LMC perf counters for stats after the test */ |
| 273 | uint64_t start_dram_dclk[BDK_NUMA_MAX_NODES][4]; |
| 274 | uint64_t start_dram_ops[BDK_NUMA_MAX_NODES][4]; |
| 275 | uint64_t stop_dram_dclk[BDK_NUMA_MAX_NODES][4]; |
| 276 | uint64_t stop_dram_ops[BDK_NUMA_MAX_NODES][4]; |
| 277 | for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) |
| 278 | { |
| 279 | if (flags & (1 << node)) |
| 280 | { |
| 281 | const int num_dram_controllers = __bdk_dram_get_num_lmc(node); |
| 282 | for (int i = 0; i < num_dram_controllers; i++) |
| 283 | { |
| 284 | start_dram_dclk[node][i] = BDK_CSR_READ(node, BDK_LMCX_DCLK_CNT(i)); |
| 285 | start_dram_ops[node][i] = BDK_CSR_READ(node, BDK_LMCX_OPS_CNT(i)); |
| 286 | } |
| 287 | } |
| 288 | } |
| 289 | /* Remember the CCPI link counters for stats after the test */ |
| 290 | uint64_t start_ccpi_data[BDK_NUMA_MAX_NODES][3]; |
| 291 | uint64_t start_ccpi_idle[BDK_NUMA_MAX_NODES][3]; |
| 292 | uint64_t start_ccpi_err[BDK_NUMA_MAX_NODES][3]; |
| 293 | uint64_t stop_ccpi_data[BDK_NUMA_MAX_NODES][3]; |
| 294 | uint64_t stop_ccpi_idle[BDK_NUMA_MAX_NODES][3]; |
| 295 | uint64_t stop_ccpi_err[BDK_NUMA_MAX_NODES][3]; |
| 296 | if (!bdk_numa_is_only_one()) |
| 297 | { |
| 298 | for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) |
| 299 | { |
| 300 | if (flags & (1 << node)) |
| 301 | { |
| 302 | for (int link = 0; link < 3; link++) |
| 303 | { |
| 304 | start_ccpi_data[node][link] = BDK_CSR_READ(node, BDK_OCX_TLKX_STAT_DATA_CNT(link)); |
| 305 | start_ccpi_idle[node][link] = BDK_CSR_READ(node, BDK_OCX_TLKX_STAT_IDLE_CNT(link)); |
| 306 | start_ccpi_err[node][link] = BDK_CSR_READ(node, BDK_OCX_TLKX_STAT_ERR_CNT(link)); |
| 307 | } |
| 308 | } |
| 309 | } |
| 310 | } |
| 311 | |
| 312 | /* WARNING: This code assumes the same memory range is being tested on |
| 313 | all nodes. The same number of cores are used on each node to test |
| 314 | its local memory */ |
| 315 | uint64_t work_address = start_address; |
| 316 | dram_test_flags = flags; |
| 317 | bdk_atomic_set64(&dram_test_thread_errors, 0); |
| 318 | while ((work_address < end_address) && ((dram_test_thread_errors == 0) || (flags & BDK_DRAM_TEST_NO_STOP_ERROR))) |
| 319 | { |
| 320 | /* Check at most MAX_CHUNK_SIZE across each iteration. We only report |
| 321 | progress between chunks, so keep them reasonably small */ |
| 322 | const uint64_t MAX_CHUNK_SIZE = 1ull << 28; /* 256MB */ |
| 323 | uint64_t size = end_address - work_address; |
| 324 | if (size > MAX_CHUNK_SIZE) |
| 325 | size = MAX_CHUNK_SIZE; |
| 326 | |
| 327 | /* Divide memory evenly between the cores. Round the size up so that |
| 328 | all memory is covered. The last core may have slightly less memory to |
| 329 | test */ |
| 330 | uint64_t thread_size = (size + (max_cores - 1)) / max_cores; |
| 331 | thread_size += 127; |
| 332 | thread_size &= -128; |
| 333 | dram_test_thread_start = work_address; |
| 334 | dram_test_thread_end = work_address + size; |
| 335 | dram_test_thread_size = thread_size; |
| 336 | BDK_WMB; |
| 337 | |
| 338 | /* Poke the watchdog */ |
| 339 | BDK_CSR_WRITE(bdk_numa_local(), BDK_GTI_CWD_POKEX(0), 0); |
| 340 | |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 341 | /* disable progress output when batch mode is ON */ |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 342 | if (!(flags & BDK_DRAM_TEST_NO_PROGRESS)) { |
| 343 | |
| 344 | /* Report progress percentage */ |
| 345 | int percent_x10 = (work_address - start_address) * 1000 / (end_address - start_address); |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 346 | printf(" %3d.%d%% complete, testing [0x%011llx:0x%011llx]\r", |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 347 | percent_x10 / 10, percent_x10 % 10, work_address, work_address + size - 1); |
| 348 | fflush(stdout); |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 349 | } |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 350 | |
| 351 | work_address += size; |
| 352 | |
| 353 | /* Start threads for all the cores */ |
| 354 | int total_count = 0; |
| 355 | bdk_atomic_set64(&dram_test_thread_done, 0); |
| 356 | for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) |
| 357 | { |
| 358 | if (flags & (1 << node)) |
| 359 | { |
| 360 | const int num_cores = bdk_get_num_cores(node); |
| 361 | int per_node = 0; |
| 362 | for (int core = 0; core < num_cores; core++) |
| 363 | { |
| 364 | if (per_node >= max_cores) |
| 365 | break; |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 366 | BDK_TRACE(DRAM_TEST, "Starting thread %d on node %d for memory test\n", per_node, node); |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 367 | dram_test_thread(per_node, (void *)test_info); |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 368 | } |
| 369 | } |
| 370 | } |
| 371 | |
| 372 | #if 0 |
| 373 | /* Wait for threads to finish */ |
| 374 | while (bdk_atomic_get64(&dram_test_thread_done) < total_count) |
| 375 | bdk_thread_yield(); |
| 376 | #else |
| 377 | #define TIMEOUT_SECS 30 // FIXME: long enough so multicore RXOR 224 should not print out |
| 378 | /* Wait for threads to finish, with progress */ |
| 379 | int cur_count; |
| 380 | uint64_t cur_time; |
Peter Lemenkov | 7bbe3bb | 2018-12-07 11:23:21 +0100 | [diff] [blame] | 381 | uint64_t period = bdk_clock_get_rate(bdk_numa_local(), BDK_CLOCK_TIME) * TIMEOUT_SECS; // FIXME? |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 382 | uint64_t timeout = bdk_clock_get_count(BDK_CLOCK_TIME) + period; |
| 383 | do { |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 384 | cur_count = bdk_atomic_get64(&dram_test_thread_done); |
| 385 | cur_time = bdk_clock_get_count(BDK_CLOCK_TIME); |
| 386 | if (cur_time >= timeout) { |
| 387 | BDK_TRACE(DRAM_TEST, "N%d: Waiting for %d cores\n", |
| 388 | bdk_numa_local(), total_count - cur_count); |
| 389 | timeout = cur_time + period; |
| 390 | } |
| 391 | } while (cur_count < total_count); |
| 392 | #endif |
| 393 | } |
| 394 | |
| 395 | /* Get the DRAM perf counters */ |
| 396 | for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) |
| 397 | { |
| 398 | if (flags & (1 << node)) |
| 399 | { |
| 400 | const int num_dram_controllers = __bdk_dram_get_num_lmc(node); |
| 401 | for (int i = 0; i < num_dram_controllers; i++) |
| 402 | { |
| 403 | stop_dram_dclk[node][i] = BDK_CSR_READ(node, BDK_LMCX_DCLK_CNT(i)); |
| 404 | stop_dram_ops[node][i] = BDK_CSR_READ(node, BDK_LMCX_OPS_CNT(i)); |
| 405 | } |
| 406 | } |
| 407 | } |
| 408 | /* Get the CCPI link counters */ |
| 409 | if (!bdk_numa_is_only_one()) |
| 410 | { |
| 411 | for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) |
| 412 | { |
| 413 | if (flags & (1 << node)) |
| 414 | { |
| 415 | for (int link = 0; link < 3; link++) |
| 416 | { |
| 417 | stop_ccpi_data[node][link] = BDK_CSR_READ(node, BDK_OCX_TLKX_STAT_DATA_CNT(link)); |
| 418 | stop_ccpi_idle[node][link] = BDK_CSR_READ(node, BDK_OCX_TLKX_STAT_IDLE_CNT(link)); |
| 419 | stop_ccpi_err[node][link] = BDK_CSR_READ(node, BDK_OCX_TLKX_STAT_ERR_CNT(link)); |
| 420 | } |
| 421 | } |
| 422 | } |
| 423 | } |
| 424 | |
| 425 | /* disable progress output when batch mode is ON */ |
| 426 | if (!(flags & BDK_DRAM_TEST_NO_PROGRESS)) { |
| 427 | |
| 428 | /* Report progress percentage as complete */ |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 429 | printf(" %3d.%d%% complete, testing [0x%011llx:0x%011llx]\n", |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 430 | 100, 0, start_address, end_address - 1); |
| 431 | fflush(stdout); |
| 432 | } |
| 433 | |
| 434 | if (!(flags & BDK_DRAM_TEST_NO_STATS)) |
| 435 | { |
| 436 | /* Display LMC load */ |
| 437 | for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) |
| 438 | { |
| 439 | if (flags & (1 << node)) |
| 440 | { |
| 441 | const int num_dram_controllers = __bdk_dram_get_num_lmc(node); |
| 442 | for (int i = 0; i < num_dram_controllers; i++) |
| 443 | { |
| 444 | uint64_t ops = stop_dram_ops[node][i] - start_dram_ops[node][i]; |
| 445 | uint64_t dclk = stop_dram_dclk[node][i] - start_dram_dclk[node][i]; |
| 446 | if (dclk == 0) |
| 447 | dclk = 1; |
| 448 | uint64_t percent_x10 = ops * 1000 / dclk; |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 449 | printf(" Node %d, LMC%d: ops %llu, cycles %llu, used %llu.%llu%%\n", |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 450 | node, i, ops, dclk, percent_x10 / 10, percent_x10 % 10); |
| 451 | } |
| 452 | } |
| 453 | } |
| 454 | if (flags & BDK_DRAM_TEST_USE_CCPI) |
| 455 | { |
| 456 | /* Display CCPI load */ |
| 457 | for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) |
| 458 | { |
| 459 | if (flags & (1 << node)) |
| 460 | { |
| 461 | for (int link = 0; link < 3; link++) |
| 462 | { |
| 463 | uint64_t busy = stop_ccpi_data[node][link] - start_ccpi_data[node][link]; |
| 464 | busy += stop_ccpi_err[node][link] - start_ccpi_err[node][link]; |
| 465 | uint64_t total = stop_ccpi_idle[node][link] - start_ccpi_idle[node][link]; |
| 466 | total += busy; |
| 467 | if (total == 0) |
| 468 | continue; |
| 469 | uint64_t percent_x10 = busy * 1000 / total; |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 470 | printf(" Node %d, CCPI%d: busy %llu, total %llu, used %llu.%llu%%\n", |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 471 | node, link, busy, total, percent_x10 / 10, percent_x10 % 10); |
| 472 | } |
| 473 | } |
| 474 | } |
| 475 | } |
| 476 | } |
| 477 | return dram_test_thread_errors; |
| 478 | } |
| 479 | |
| 480 | /** |
| 481 | * Perform a memory test. |
| 482 | * |
| 483 | * @param test Test type to run |
| 484 | * @param start_address |
| 485 | * Physical address to start at |
| 486 | * @param length Length of memory block |
| 487 | * @param flags Flags to control memory test options. Zero defaults to testing all |
| 488 | * node with statistics and progress output. |
| 489 | * |
| 490 | * @return Number of errors found. Zero is success. Negative means the test |
| 491 | * did not run due to some other failure. |
| 492 | */ |
| 493 | int bdk_dram_test(int test, uint64_t start_address, uint64_t length, bdk_dram_test_flags_t flags) |
| 494 | { |
| 495 | /* These limits are arbitrary. They just make sure we aren't doing something |
| 496 | silly, like test a non cache line aligned memory region */ |
| 497 | if (start_address & 0xffff) |
| 498 | { |
| 499 | bdk_error("DRAM test start address must be aligned on a 64KB boundary\n"); |
| 500 | return -1; |
| 501 | } |
| 502 | if (length & 0xffff) |
| 503 | { |
| 504 | bdk_error("DRAM test length must be a multiple of 64KB\n"); |
| 505 | return -1; |
| 506 | } |
| 507 | |
| 508 | const char *name = bdk_dram_get_test_name(test); |
| 509 | if (name == NULL) |
| 510 | { |
| 511 | bdk_error("Invalid DRAM test number %d\n", test); |
| 512 | return -1; |
| 513 | } |
| 514 | |
| 515 | /* If no nodes are selected assume the user meant all nodes */ |
| 516 | if ((flags & (BDK_DRAM_TEST_NODE0 | BDK_DRAM_TEST_NODE1 | BDK_DRAM_TEST_NODE2 | BDK_DRAM_TEST_NODE3)) == 0) |
| 517 | flags |= BDK_DRAM_TEST_NODE0 | BDK_DRAM_TEST_NODE1 | BDK_DRAM_TEST_NODE2 | BDK_DRAM_TEST_NODE3; |
| 518 | |
| 519 | /* Remove nodes from the flags that don't exist */ |
| 520 | for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) |
| 521 | { |
| 522 | if (flags & BDK_DRAM_TEST_USE_CCPI) |
| 523 | { |
| 524 | if (!bdk_numa_exists(node ^ 1)) |
| 525 | flags &= ~(1 << node); |
| 526 | } |
| 527 | else |
| 528 | { |
| 529 | if (!bdk_numa_exists(node)) |
| 530 | flags &= ~(1 << node); |
| 531 | } |
| 532 | } |
| 533 | |
| 534 | |
| 535 | /* Make sure the start address is higher that the BDK's active range */ |
| 536 | uint64_t top_of_bdk = bdk_dram_get_top_of_bdk(); |
| 537 | if (start_address < top_of_bdk) |
| 538 | start_address = top_of_bdk; |
| 539 | |
| 540 | /* Clear ECC error counters before starting the test */ |
| 541 | for (int chan = 0; chan < BDK_MAX_MEM_CHANS; chan++) { |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 542 | bdk_atomic_set64(&__bdk_dram_ecc_single_bit_errors[chan], 0); |
| 543 | bdk_atomic_set64(&__bdk_dram_ecc_double_bit_errors[chan], 0); |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 544 | } |
| 545 | |
| 546 | /* Make sure at least one core from each node is running */ |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 547 | /* FIXME(dhendrix): we only care about core0 on node0 for now */ |
| 548 | #if 0 |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 549 | for (bdk_node_t node = BDK_NODE_0; node < BDK_NUMA_MAX_NODES; node++) |
| 550 | { |
| 551 | if (flags & (1<<node)) |
| 552 | { |
| 553 | int use_node = (flags & BDK_DRAM_TEST_USE_CCPI) ? node ^ 1 : node; |
| 554 | if (bdk_get_running_coremask(use_node) == 0) |
| 555 | bdk_init_cores(use_node, 1); |
| 556 | } |
| 557 | } |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 558 | #endif |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 559 | |
| 560 | /* This returns any data compare errors found */ |
| 561 | int errors = __bdk_dram_run_test(&TEST_INFO[test], start_address, length, flags); |
| 562 | |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 563 | /* Check ECC error counters after the test */ |
| 564 | int64_t ecc_single = 0; |
| 565 | int64_t ecc_double = 0; |
| 566 | int64_t ecc_single_errs[BDK_MAX_MEM_CHANS]; |
| 567 | int64_t ecc_double_errs[BDK_MAX_MEM_CHANS]; |
| 568 | |
| 569 | for (int chan = 0; chan < BDK_MAX_MEM_CHANS; chan++) { |
| 570 | ecc_single += (ecc_single_errs[chan] = bdk_atomic_get64(&__bdk_dram_ecc_single_bit_errors[chan])); |
| 571 | ecc_double += (ecc_double_errs[chan] = bdk_atomic_get64(&__bdk_dram_ecc_double_bit_errors[chan])); |
| 572 | } |
| 573 | |
| 574 | /* Always print any ECC errors */ |
| 575 | if (ecc_single || ecc_double) |
| 576 | { |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 577 | printf("Test \"%s\": ECC errors, %lld/%lld/%lld/%lld corrected, %lld/%lld/%lld/%lld uncorrected\n", |
| 578 | name, |
| 579 | ecc_single_errs[0], ecc_single_errs[1], ecc_single_errs[2], ecc_single_errs[3], |
| 580 | ecc_double_errs[0], ecc_double_errs[1], ecc_double_errs[2], ecc_double_errs[3]); |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 581 | } |
| 582 | if (errors || ecc_double || ecc_single) { |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 583 | printf("Test \"%s\": FAIL: %lld single, %lld double, %d compare errors\n", |
| 584 | name, ecc_single, ecc_double, errors); |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 585 | } |
| 586 | else |
| 587 | BDK_TRACE(DRAM_TEST, "Test \"%s\": PASS\n", name); |
| 588 | |
| 589 | return (errors + ecc_double + ecc_single); |
| 590 | } |
| 591 | |
| 592 | /** |
| 593 | * Report a DRAM address in decoded format. |
| 594 | * |
| 595 | * @param address Physical address the error occurred at |
| 596 | * |
| 597 | */ |
| 598 | static void __bdk_dram_report_address_decode(uint64_t address, char *buffer, int len) |
| 599 | { |
| 600 | int node, lmc, dimm, prank, lrank, bank, row, col; |
| 601 | |
| 602 | bdk_dram_address_extract_info(address, &node, &lmc, &dimm, &prank, &lrank, &bank, &row, &col); |
| 603 | |
| 604 | snprintf(buffer, len, "[0x%011lx] (N%d,LMC%d,DIMM%d,Rank%d/%d,Bank%02d,Row 0x%05x,Col 0x%04x)", |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 605 | address, node, lmc, dimm, prank, lrank, bank, row, col); |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 606 | } |
| 607 | |
| 608 | /** |
| 609 | * Report a DRAM address in a new decoded format. |
| 610 | * |
| 611 | * @param address Physical address the error occurred at |
| 612 | * @param xor XOR of data read vs expected data |
| 613 | * |
| 614 | */ |
| 615 | static void __bdk_dram_report_address_decode_new(uint64_t address, uint64_t orig_xor, char *buffer, int len) |
| 616 | { |
| 617 | int node, lmc, dimm, prank, lrank, bank, row, col; |
| 618 | |
| 619 | int byte = 8; // means no byte-lanes in error, should not happen |
| 620 | uint64_t bits, print_bits = 0; |
| 621 | uint64_t xor = orig_xor; |
| 622 | |
| 623 | // find the byte-lane(s) with errors |
| 624 | for (int i = 0; i < 8; i++) { |
| 625 | bits = xor & 0xffULL; |
| 626 | xor >>= 8; |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 627 | if (bits) { |
| 628 | if (byte != 8) { |
| 629 | byte = 9; // means more than 1 byte-lane was present |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 630 | print_bits = orig_xor; // print the full original |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 631 | break; // quit now |
| 632 | } else { |
| 633 | byte = i; // keep checking |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 634 | print_bits = bits; |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 635 | } |
| 636 | } |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 637 | } |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 638 | |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 639 | bdk_dram_address_extract_info(address, &node, &lmc, &dimm, &prank, &lrank, &bank, &row, &col); |
| 640 | |
| 641 | snprintf(buffer, len, "N%d.LMC%d: CMP byte %d xor 0x%02lx (DIMM%d,Rank%d/%d,Bank%02d,Row 0x%05x,Col 0x%04x)[0x%011lx]", |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 642 | node, lmc, byte, print_bits, dimm, prank, lrank, bank, row, col, address); |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 643 | } |
| 644 | |
| 645 | /** |
| 646 | * Report a DRAM error. Errors are not shown after MAX_ERRORS_TO_REPORT is |
| 647 | * exceeded. Used when a single address is involved in the failure. |
| 648 | * |
| 649 | * @param address Physical address the error occurred at |
| 650 | * @param data Data read from memory |
| 651 | * @param correct Correct data |
| 652 | * @param burst Which burst this is from, informational only |
Peter Lemenkov | 7bbe3bb | 2018-12-07 11:23:21 +0100 | [diff] [blame] | 653 | * @param fails -1 for no retries done, >= 0 number of failures during retries |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 654 | * |
| 655 | * @return Zero if a message was logged, non-zero if the error limit has been reached |
| 656 | */ |
| 657 | void __bdk_dram_report_error(uint64_t address, uint64_t data, uint64_t correct, int burst, int fails) |
| 658 | { |
| 659 | char buffer[128]; |
| 660 | char failbuf[32]; |
| 661 | int64_t errors = bdk_atomic_fetch_and_add64(&dram_test_thread_errors, 1); |
| 662 | uint64_t xor = data ^ correct; |
| 663 | |
| 664 | if (errors < MAX_ERRORS_TO_REPORT) |
| 665 | { |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 666 | if (fails < 0) { |
| 667 | snprintf(failbuf, sizeof(failbuf), " "); |
| 668 | } else { |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 669 | int percent_x10 = fails * 1000 / RETRY_LIMIT; |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 670 | snprintf(failbuf, sizeof(failbuf), ", retries failed %3d.%d%%", |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 671 | percent_x10 / 10, percent_x10 % 10); |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 672 | } |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 673 | |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 674 | __bdk_dram_report_address_decode_new(address, xor, buffer, sizeof(buffer)); |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 675 | bdk_error("%s%s\n", buffer, failbuf); |
| 676 | |
| 677 | if (errors == MAX_ERRORS_TO_REPORT-1) |
| 678 | bdk_error("No further DRAM errors will be reported\n"); |
| 679 | } |
| 680 | return; |
| 681 | } |
| 682 | |
| 683 | /** |
| 684 | * Report a DRAM error. Errors are not shown after MAX_ERRORS_TO_REPORT is |
| 685 | * exceeded. Used when two addresses might be involved in the failure. |
| 686 | * |
| 687 | * @param address1 First address involved in the failure |
| 688 | * @param data1 Data from the first address |
| 689 | * @param address2 Second address involved in the failure |
| 690 | * @param data2 Data from second address |
| 691 | * @param burst Which burst this is from, informational only |
Peter Lemenkov | 7bbe3bb | 2018-12-07 11:23:21 +0100 | [diff] [blame] | 692 | * @param fails -1 for no retries done, >= 0 number of failures during retries |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 693 | * |
| 694 | * @return Zero if a message was logged, non-zero if the error limit has been reached |
| 695 | */ |
| 696 | void __bdk_dram_report_error2(uint64_t address1, uint64_t data1, uint64_t address2, uint64_t data2, |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 697 | int burst, int fails) |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 698 | { |
| 699 | int64_t errors = bdk_atomic_fetch_and_add64(&dram_test_thread_errors, 1); |
| 700 | if (errors < MAX_ERRORS_TO_REPORT) |
| 701 | { |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 702 | char buffer1[80], buffer2[80]; |
| 703 | char failbuf[32]; |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 704 | |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 705 | if (fails < 0) { |
| 706 | snprintf(failbuf, sizeof(failbuf), " "); |
| 707 | } else { |
| 708 | snprintf(failbuf, sizeof(failbuf), ", retried %d failed %d", RETRY_LIMIT, fails); |
| 709 | } |
| 710 | __bdk_dram_report_address_decode(address1, buffer1, sizeof(buffer1)); |
| 711 | __bdk_dram_report_address_decode(address2, buffer2, sizeof(buffer2)); |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 712 | |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 713 | bdk_error("compare: data1: 0x%016llx, xor: 0x%016llx%s\n" |
| 714 | " %s\n %s\n", |
| 715 | data1, data1 ^ data2, failbuf, |
| 716 | buffer1, buffer2); |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 717 | |
| 718 | if (errors == MAX_ERRORS_TO_REPORT-1) |
| 719 | bdk_error("No further DRAM errors will be reported\n"); |
| 720 | } |
| 721 | return; |
| 722 | } |
| 723 | |
| 724 | /* Report the circumstances of a failure and try re-reading the memory |
| 725 | * location to see if the error is transient or permanent. |
| 726 | * |
| 727 | * Note: re-reading requires using evicting addresses |
| 728 | */ |
| 729 | int __bdk_dram_retry_failure(int burst, uint64_t address, uint64_t data, uint64_t expected) |
| 730 | { |
| 731 | int refail = 0; |
| 732 | |
| 733 | // bypass the retries if we are already over the limit... |
| 734 | if (bdk_atomic_get64(&dram_test_thread_errors) < MAX_ERRORS_TO_REPORT) { |
| 735 | |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 736 | /* Try re-reading the memory location. A transient error may fail |
| 737 | * on one read and work on another. Keep on retrying even when a |
| 738 | * read succeeds. |
| 739 | */ |
| 740 | for (int i = 0; i < RETRY_LIMIT; i++) { |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 741 | |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 742 | __bdk_dram_flush_to_mem(address); |
| 743 | BDK_DCACHE_INVALIDATE; |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 744 | |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 745 | uint64_t new = __bdk_dram_read64(address); |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 746 | |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 747 | if (new != expected) { |
| 748 | refail++; |
| 749 | } |
| 750 | } |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 751 | } else |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 752 | refail = -1; |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 753 | |
| 754 | // this will increment the errors always, but maybe not print... |
| 755 | __bdk_dram_report_error(address, data, expected, burst, refail); |
| 756 | |
| 757 | return 1; |
| 758 | } |
| 759 | |
| 760 | /** |
| 761 | * retry_failure2 |
| 762 | * |
| 763 | * @param burst |
| 764 | * @param address1 |
| 765 | * @param address2 |
| 766 | */ |
| 767 | int __bdk_dram_retry_failure2(int burst, uint64_t address1, uint64_t data1, uint64_t address2, uint64_t data2) |
| 768 | { |
| 769 | int refail = 0; |
| 770 | |
| 771 | // bypass the retries if we are already over the limit... |
| 772 | if (bdk_atomic_get64(&dram_test_thread_errors) < MAX_ERRORS_TO_REPORT) { |
| 773 | |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 774 | for (int i = 0; i < RETRY_LIMIT; i++) { |
| 775 | __bdk_dram_flush_to_mem(address1); |
| 776 | __bdk_dram_flush_to_mem(address2); |
| 777 | BDK_DCACHE_INVALIDATE; |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 778 | |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 779 | uint64_t d1 = __bdk_dram_read64(address1); |
| 780 | uint64_t d2 = __bdk_dram_read64(address2); |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 781 | |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 782 | if (d1 != d2) { |
| 783 | refail++; |
| 784 | } |
| 785 | } |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 786 | } else |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 787 | refail = -1; |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 788 | |
| 789 | // this will increment the errors always, but maybe not print... |
| 790 | __bdk_dram_report_error2(address1, data1, address2, data2, burst, refail); |
| 791 | |
| 792 | return 1; |
| 793 | } |
| 794 | |
| 795 | /** |
| 796 | * Inject a DRAM error at a specific address in memory. The injection can either |
| 797 | * be a single bit inside the byte, or a double bit error in the ECC byte. Double |
| 798 | * bit errors may corrupt memory, causing software to crash. The corruption is |
| 799 | * written to memory and will continue to exist until the cache line is written |
| 800 | * again. After a call to this function, the BDK should report a ECC error. Double |
| 801 | * bit errors corrupt bits 0-1. |
| 802 | * |
| 803 | * @param address Physical address to corrupt. Any byte alignment is supported |
| 804 | * @param bit Bit to corrupt in the byte (0-7), or -1 to create a double bit fault in the ECC |
| 805 | * byte. |
| 806 | */ |
| 807 | void bdk_dram_test_inject_error(uint64_t address, int bit) |
| 808 | { |
| 809 | uint64_t aligned_address = address & -16; |
| 810 | int corrupt_bit = -1; |
| 811 | if (bit >= 0) |
| 812 | corrupt_bit = (address & 0xf) * 8 + bit; |
| 813 | |
| 814 | /* Extract the DRAM controller information */ |
| 815 | int node, lmc, dimm, prank, lrank, bank, row, col; |
| 816 | bdk_dram_address_extract_info(address, &node, &lmc, &dimm, &prank, &lrank, &bank, &row, &col); |
| 817 | |
| 818 | /* Read the current data */ |
| 819 | uint64_t data = __bdk_dram_read64(aligned_address); |
| 820 | |
| 821 | /* Program LMC to inject the error */ |
| 822 | if ((corrupt_bit >= 0) && (corrupt_bit < 64)) |
| 823 | BDK_CSR_WRITE(node, BDK_LMCX_CHAR_MASK0(lmc), 1ull << corrupt_bit); |
| 824 | else if (bit == -1) |
| 825 | BDK_CSR_WRITE(node, BDK_LMCX_CHAR_MASK0(lmc), 3); |
| 826 | else |
| 827 | BDK_CSR_WRITE(node, BDK_LMCX_CHAR_MASK0(lmc), 0); |
| 828 | if (corrupt_bit >= 64) |
| 829 | BDK_CSR_WRITE(node, BDK_LMCX_CHAR_MASK2(lmc), 1ull << (corrupt_bit - 64)); |
| 830 | else |
| 831 | BDK_CSR_WRITE(node, BDK_LMCX_CHAR_MASK2(lmc), 0); |
| 832 | BDK_CSR_MODIFY(c, node, BDK_LMCX_ECC_PARITY_TEST(lmc), |
| 833 | c.s.ecc_corrupt_idx = (address & 0x7f) >> 4; |
| 834 | c.s.ecc_corrupt_ena = 1); |
| 835 | BDK_CSR_READ(node, BDK_LMCX_ECC_PARITY_TEST(lmc)); |
| 836 | |
| 837 | /* Perform a write and push it to DRAM. This creates the error */ |
| 838 | __bdk_dram_write64(aligned_address, data); |
| 839 | __bdk_dram_flush_to_mem(aligned_address); |
| 840 | |
| 841 | /* Disable error injection */ |
| 842 | BDK_CSR_MODIFY(c, node, BDK_LMCX_ECC_PARITY_TEST(lmc), |
| 843 | c.s.ecc_corrupt_ena = 0); |
| 844 | BDK_CSR_READ(node, BDK_LMCX_ECC_PARITY_TEST(lmc)); |
| 845 | BDK_CSR_WRITE(node, BDK_LMCX_CHAR_MASK0(lmc), 0); |
| 846 | BDK_CSR_WRITE(node, BDK_LMCX_CHAR_MASK2(lmc), 0); |
| 847 | |
| 848 | /* Read back the data, which should now cause an error */ |
David Hendricks | 7d48ac5 | 2018-03-09 14:30:38 -0800 | [diff] [blame] | 849 | printf("Loading the injected error address 0x%llx, node=%d, lmc=%d, dimm=%d, rank=%d/%d, bank=%d, row=%d, col=%d\n", |
David Hendricks | 2004b93 | 2018-03-09 13:58:27 -0800 | [diff] [blame] | 850 | address, node, lmc, dimm, prank, lrank, bank, row, col); |
| 851 | __bdk_dram_read64(aligned_address); |
| 852 | } |