| /* SPDX-License-Identifier: GPL-2.0-only */ |
| /* This file is part of the coreboot project. */ |
| /* |
| * This file is derived from memcpy_32.c in the Linux kernel. |
| */ |
| |
| #include <string.h> |
| |
| void *memmove(void *dest, const void *src, size_t n) |
| { |
| int d0, d1, d2, d3, d4, d5; |
| char *ret = dest; |
| |
| __asm__ __volatile__( |
| /* Handle more 16bytes in loop */ |
| "cmp $0x10, %0\n\t" |
| "jb 1f\n\t" |
| |
| /* Decide forward/backward copy mode */ |
| "cmp %2, %1\n\t" |
| "jb 2f\n\t" |
| |
| /* |
| * movs instruction have many startup latency |
| * so we handle small size by general register. |
| */ |
| "cmp $680, %0\n\t" |
| "jb 3f\n\t" |
| /* |
| * movs instruction is only good for aligned case. |
| */ |
| "mov %1, %3\n\t" |
| "xor %2, %3\n\t" |
| "and $0xff, %3\n\t" |
| "jz 4f\n\t" |
| "3:\n\t" |
| "sub $0x10, %0\n\t" |
| |
| /* |
| * We gobble 16byts forward in each loop. |
| */ |
| "3:\n\t" |
| "sub $0x10, %0\n\t" |
| "mov 0*4(%1), %3\n\t" |
| "mov 1*4(%1), %4\n\t" |
| "mov %3, 0*4(%2)\n\t" |
| "mov %4, 1*4(%2)\n\t" |
| "mov 2*4(%1), %3\n\t" |
| "mov 3*4(%1), %4\n\t" |
| "mov %3, 2*4(%2)\n\t" |
| "mov %4, 3*4(%2)\n\t" |
| "lea 0x10(%1), %1\n\t" |
| "lea 0x10(%2), %2\n\t" |
| "jae 3b\n\t" |
| "add $0x10, %0\n\t" |
| "jmp 1f\n\t" |
| |
| /* |
| * Handle data forward by movs. |
| */ |
| ".p2align 4\n\t" |
| "4:\n\t" |
| "mov -4(%1, %0), %3\n\t" |
| "lea -4(%2, %0), %4\n\t" |
| "shr $2, %0\n\t" |
| "rep movsl\n\t" |
| "mov %3, (%4)\n\t" |
| "jmp 11f\n\t" |
| /* |
| * Handle data backward by movs. |
| */ |
| ".p2align 4\n\t" |
| "6:\n\t" |
| "mov (%1), %3\n\t" |
| "mov %2, %4\n\t" |
| "lea -4(%1, %0), %1\n\t" |
| "lea -4(%2, %0), %2\n\t" |
| "shr $2, %0\n\t" |
| "std\n\t" |
| "rep movsl\n\t" |
| "mov %3,(%4)\n\t" |
| "cld\n\t" |
| "jmp 11f\n\t" |
| |
| /* |
| * Start to prepare for backward copy. |
| */ |
| ".p2align 4\n\t" |
| "2:\n\t" |
| "cmp $680, %0\n\t" |
| "jb 5f\n\t" |
| "mov %1, %3\n\t" |
| "xor %2, %3\n\t" |
| "and $0xff, %3\n\t" |
| "jz 6b\n\t" |
| |
| /* |
| * Calculate copy position to tail. |
| */ |
| "5:\n\t" |
| "add %0, %1\n\t" |
| "add %0, %2\n\t" |
| "sub $0x10, %0\n\t" |
| |
| /* |
| * We gobble 16byts backward in each loop. |
| */ |
| "7:\n\t" |
| "sub $0x10, %0\n\t" |
| |
| "mov -1*4(%1), %3\n\t" |
| "mov -2*4(%1), %4\n\t" |
| "mov %3, -1*4(%2)\n\t" |
| "mov %4, -2*4(%2)\n\t" |
| "mov -3*4(%1), %3\n\t" |
| "mov -4*4(%1), %4\n\t" |
| "mov %3, -3*4(%2)\n\t" |
| "mov %4, -4*4(%2)\n\t" |
| "lea -0x10(%1), %1\n\t" |
| "lea -0x10(%2), %2\n\t" |
| "jae 7b\n\t" |
| /* |
| * Calculate copy position to head. |
| */ |
| "add $0x10, %0\n\t" |
| "sub %0, %1\n\t" |
| "sub %0, %2\n\t" |
| |
| /* |
| * Move data from 8 bytes to 15 bytes. |
| */ |
| ".p2align 4\n\t" |
| "1:\n\t" |
| "cmp $8, %0\n\t" |
| "jb 8f\n\t" |
| "mov 0*4(%1), %3\n\t" |
| "mov 1*4(%1), %4\n\t" |
| "mov -2*4(%1, %0), %5\n\t" |
| "mov -1*4(%1, %0), %1\n\t" |
| |
| "mov %3, 0*4(%2)\n\t" |
| "mov %4, 1*4(%2)\n\t" |
| "mov %5, -2*4(%2, %0)\n\t" |
| "mov %1, -1*4(%2, %0)\n\t" |
| "jmp 11f\n\t" |
| |
| /* |
| * Move data from 4 bytes to 7 bytes. |
| */ |
| ".p2align 4\n\t" |
| "8:\n\t" |
| "cmp $4, %0\n\t" |
| "jb 9f\n\t" |
| "mov 0*4(%1), %3\n\t" |
| "mov -1*4(%1, %0), %4\n\t" |
| "mov %3, 0*4(%2)\n\t" |
| "mov %4, -1*4(%2, %0)\n\t" |
| "jmp 11f\n\t" |
| |
| /* |
| * Move data from 2 bytes to 3 bytes. |
| */ |
| ".p2align 4\n\t" |
| "9:\n\t" |
| "cmp $2, %0\n\t" |
| "jb 10f\n\t" |
| "movw 0*2(%1), %%dx\n\t" |
| "movw -1*2(%1, %0), %%bx\n\t" |
| "movw %%dx, 0*2(%2)\n\t" |
| "movw %%bx, -1*2(%2, %0)\n\t" |
| "jmp 11f\n\t" |
| |
| /* |
| * Move data for 1 byte. |
| */ |
| ".p2align 4\n\t" |
| "10:\n\t" |
| "cmp $1, %0\n\t" |
| "jb 11f\n\t" |
| "movb (%1), %%cl\n\t" |
| "movb %%cl, (%2)\n\t" |
| ".p2align 4\n\t" |
| "11:" |
| : "=&c" (d0), "=&S" (d1), "=&D" (d2), |
| "=r" (d3), "=r" (d4), "=r"(d5) |
| : "0" (n), |
| "1" (src), |
| "2" (dest) |
| : "memory"); |
| |
| return ret; |
| |
| } |