path: root/sysdeps/aarch64/strcpy.S

/* strcpy/stpcpy - copy a string returning pointer to start/end.
   Copyright (C) 2013-2017 Free Software Foundation, Inc.
   This file is part of the GNU C Library.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, see
   <http://www.gnu.org/licenses/>.  */

/* To build as stpcpy, define BUILD_STPCPY before compiling this file.

   To test the page crossing code path more thoroughly, compile with
   -DSTRCPY_TEST_PAGE_CROSS - this will force all unaligned copies through
   the slower entry path.  This option is not intended for production use.  */

#include <sysdep.h>

/* Assumptions:
 *
 * ARMv8-a, AArch64, unaligned accesses, min page size 4k.
 */

/* Arguments and results.  */
#define dstin		x0
#define srcin		x1

/* Locals and temporaries.  */
#define src		x2
#define dst		x3
#define data1		x4
#define data1w		w4
#define data2		x5
#define data2w		w5
#define has_nul1	x6
#define has_nul2	x7
#define tmp1		x8
#define tmp2		x9
#define tmp3		x10
#define tmp4		x11
#define zeroones	x12
#define data1a		x13
#define data2a		x14
#define pos		x15
#define len		x16
#define to_align	x17

#ifdef BUILD_STPCPY
#define STRCPY __stpcpy
#else
#define STRCPY strcpy
#endif

	/* NUL detection works on the principle that (X - 1) & (~X) & 0x80
	   (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
	   can be done in parallel across the entire word.  */

#define REP8_01 0x0101010101010101
#define REP8_7f 0x7f7f7f7f7f7f7f7f
#define REP8_80 0x8080808080808080

	/* AArch64 systems have a minimum page size of 4k.  We can do a quick
	   page size check for crossing this boundary on entry and if we
	   do not, then we can short-circuit much of the entry code.  We
	   expect early page-crossing strings to be rare (probability of
	   16/MIN_PAGE_SIZE ~= 0.4%), so the branch should be quite
	   predictable, even with random strings.

	   We don't bother checking for larger page sizes, the cost of setting
	   up the correct page size is just not worth the extra gain from
	   a small reduction in the cases taking the slow path.  Note that
	   we only care about whether the first fetch, which may be
	   misaligned, crosses a page boundary - after that we move to aligned
	   fetches for the remainder of the string.  */

#ifdef STRCPY_TEST_PAGE_CROSS
	/* Make everything that isn't Qword aligned look like a page cross.  */
#define MIN_PAGE_P2 4
#else
#define MIN_PAGE_P2 12
#endif

#define MIN_PAGE_SIZE (1 << MIN_PAGE_P2)

ENTRY_ALIGN (STRCPY, 6)
	DELOUSE (0)
	DELOUSE (1)
	/* For moderately short strings, the fastest way to do the copy is to
	   calculate the length of the string in the same way as strlen, then
	   essentially do a memcpy of the result.  This avoids the need for
	   multiple byte copies and further means that by the time we
	   reach the bulk copy loop we know we can always use DWord
	   accesses.  We expect strcpy to rarely be called repeatedly
	   with the same source string, so branch prediction is likely to
	   always be difficult - we mitigate against this by preferring
	   conditional select operations over branches whenever this is
	   feasible.  */
	and	tmp2, srcin, #(MIN_PAGE_SIZE - 1)
	mov	zeroones, #REP8_01
	and	to_align, srcin, #15