path: root/sysdeps/powerpc/powerpc64/memcpy.S

/* Optimized memcpy implementation for PowerPC64.
   Copyright (C) 2003-2025 Free Software Foundation, Inc.
   This file is part of the GNU C Library.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, see
   <https://www.gnu.org/licenses/>.  */

#include <sysdep.h>

/* void * [r3] memcpy (void *dst [r3], void *src [r4], size_t len [r5]);
   Returns 'dst'.

   Memcpy handles short copies (< 32-bytes) using a binary move blocks
   (no loops) of lwz/stw.  The tail (remaining 1-3) bytes is handled
   with the appropriate combination of byte and halfword load/stores.
   There is minimal effort to optimize the alignment of short moves.
   The 64-bit implementations of POWER3 and POWER4 do a reasonable job
   of handling unaligned load/stores that do not cross 32-byte boundaries.

   Longer moves (>= 32-bytes) justify the effort to get at least the
   destination doubleword (8-byte) aligned.  Further optimization is
   possible when both source and destination are doubleword aligned.
   Each case has a optimized unrolled loop.   */

#ifndef MEMCPY
# define MEMCPY memcpy
#endif

ENTRY_TOCLESS (MEMCPY, 5)
	CALL_MCOUNT 3

    cmpldi cr1,5,31
    neg   0,3
    std   3,-16(1)
    std   31,-8(1)
    cfi_offset(31,-8)
    andi. 11,3,7	/* check alignment of dst.  */
    clrldi 0,0,61	/* Number of bytes until the 1st doubleword of dst.  */
    clrldi 10,4,61	/* check alignment of src.  */
    cmpldi cr6,5,8
    ble-  cr1,.L2	/* If move < 32 bytes use short move code.  */
    cmpld cr6,10,11
    mr    12,4
    srdi  9,5,3		/* Number of full double words remaining.  */
    mtcrf 0x01,0
    mr    31,5
    beq   .L0

    subf  31,0,5
  /* Move 0-7 bytes as needed to get the destination doubleword aligned.  */
1:  bf    31,2f
    lbz   6,0(12)
    addi  12,12,1
    stb   6,0(3)
    addi  3,3,1
2:  bf    30,4f
    lhz   6,0(12)
    addi  12,12,2
    sth   6,0(3)
    addi  3,3,2
4:  bf    29,0f
    lwz   6,0(12)
    addi  12,12,4
    stw   6,0(3)
    addi  3,3,4
0:
    clrldi 10,12,61	/* check alignment of src again.  */
    srdi  9,31,3	/* Number of full double words remaining.  */

  /* Copy doublewords from source to destination, assuming the
     destination is aligned on a doubleword boundary.

     At this point we know there are at least 25 bytes left (32-7) to copy.
     The next step is to determine if the source is also doubleword aligned.
     If not branch to the unaligned move code at .L6. which uses
     a load, shift, store strategy.

     Otherwise source and destination are doubleword aligned, and we can
     the optimized doubleword copy loop.  */
.L0:
    clrldi	11,31,61
    mtcrf 0x01,9
    bne-  cr6,.L6   /* If source is not DW aligned.  */

  /* Move doublewords where destination and source are DW aligned.
     Use a unrolled loop to copy 4 doubleword (32-bytes) per iteration.
     If the copy is not an exact multiple of 32 bytes, 1-3
     doublewords are copied as needed to set up the main loop.  After
     the main loop exits there may be a tail of 1-7 bytes. These byte are
     copied a word/halfword/byte at a time as needed to preserve alignment.  */

    srdi  8,31,5
    cmpldi	cr1,9,4
    cmpldi	cr6,11,0
    mr    11,12

    bf    30,1f
    ld    6,0(12)
    ld    7,8(12)
    addi  11,12,16
    mtctr 8
    std   6,0(3)
    std   7,8(3)
    addi  10,3,16
    bf    31,4f
    ld    0,16(12)
    std   0,16(3)
    blt   cr1,3f
    addi  11,12,24
    addi  10,3,24
    b     4f
    .align  4
1:
    mr    10,3
    mtctr 8
    bf    31,4f
    ld    6,0(12)
    addi  11,12,8
    std   6,0(3)
    addi  10,3,8

    .align  4
4:
    ld    6,0(11)
    ld    7,8(11)
    ld    8,16(11)
    ld    0,24(11)
    addi  11,11,32
2:
    std   6,0(10)
    std   7,8(10)
    std   8,16(10)
    std   0,24(10)
    addi  10,10,32
    bdnz  4b
3:

    rldicr 0,31,0,60
    mtcrf 0x01,31
    beq   cr6,0f
.L9:
    add   3,3,0
    add   12,12,0

/*  At this point we have a tail of 0-7 bytes and we know that the
    destination is double word aligned.  */
4:  bf    29,2f
    lwz   6,0(12)
    addi  12,12,4
    stw   6,0(3)
    addi  3,3,4
2:  bf    30,1f
    lhz   6,0(12)
    addi  12,12,2
    sth   6,0(3)
    addi  3,3,2
1:  bf    31,0f
    lbz   6,0(12)
    stb   6,0(3)
0:
  /* Return original