path: root/sysdeps/powerpc/powerpc64/power8/strnlen.S

/* Optimized strnlen implementation for POWER8 using a vmx loop.

   Copyright (C) 2017-2025 Free Software Foundation, Inc.
   This file is part of the GNU C Library.
   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.
   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.
   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, see
   <https://www.gnu.org/licenses/>.  */

/* It is implemented the following heuristic:
	1. Case maxlen <= 32: align the pointer to 8 bytes to loop through
	reading doublewords. Uses the POWER7 algorithm.
	2. Case maxlen > 32: check for null bytes in the first 16 bytes using
	unaligned accesses. Return length if found. Otherwise:
		2.1 Case maxlen < 64: deduct the bytes previously read, align
		the pointer to 16 bytes and loop through reading quadwords
		until find null bytes or reach maxlen.
		2.2 Case maxlen > 64: deduct the bytes previously read, align
		the pointer to 64 bytes and set up a counter to loop through
		reading in strides of 64 bytes. In case it finished the loop
		with null bytes not found, process the remainder bytes by
		switching to the loop to heuristic in 2.1.  */

#include <sysdep.h>

/* Define default page size to 4KB.  */
#define PAGE_SIZE 4096


/* int [r3] strnlen (char *s [r3], size_t maxlen [r4])  */
	.machine  power8
ENTRY_TOCLESS (__strnlen)
	CALL_MCOUNT 2
	dcbt	0,r3

	cmpldi	r4,32           /* Check if maxlen <= 32.  */
	ble	L(small_range)  /* If maxlen <= 32.  */

	/* Upcoming 16 bytes unaligned accesses cannot cross the page boundary
	   otherwise the processor throws an memory access error.
	   Use following code to check there is room for such as accesses:
	     (((size_t) s) % PAGE_SIZE > (PAGE_SIZE - 16)
	   If it is disallowed then switch to the code that handles
	   the string when maxlen <= 32.  */
	clrldi	r10,r3,52
	cmpldi  cr7,r10,PAGE_SIZE-16
	bgt     cr7,L(small_range)	/* If less than 16B of page end.  */

	/* Compute our permute constant r8.  */
	li	r7,0
	/* Compute a bpermd constant to move bit 0 of each word into
	   a halfword value, and count trailing zeros.  */
#ifdef __LITTLE_ENDIAN__
	li	r8,0x2820
	oris	r8,r8,0x3830
	sldi	r8,r8,32
	ori	r8,r8,0x0800
	oris	r8,r8,0x1810
#else
	li	r8,0x1018
	oris	r8,r8,0x0008
	sldi	r8,r8,32
	ori	r8,r8,0x3038
	oris	r8,r8,0x2028
#endif

	/* maxlen > 32. Optimistically check for null bytes in the first
	   16 bytes of the string using unaligned accesses.  */
	ld	r5,0(r3)
	ld	r6,8(r3)
	cmpb	r10,r7,r5		/* Check for null bytes in DWORD1.  */
	cmpb	r11,r7,r6		/* Check for null bytes in DWORD2.  */
	or.	r7,r10,r11
	bne	cr0, L(early_find)	/* If found null bytes.  */

	/* At this point maxlen > 32 and null bytes were not found at first
	   16 bytes. Prepare for loop using VMX.  */

	/* r3 == s, r4 == maxlen. All other volatile regs are unused now.  */

	addi	r5,r3,16	/* Align up, or just add the 16B we
				   already checked.  */
	li	r0,15
	and	r7,r5,r0	/* Find offset into 16B alignment.  */
	andc	r5,r5,r0	/* Quadword align up s to the next quadword.  */
	li	r0,16
	subf	r0,r7,r0
	subf	r4,r0,r4	/* Deduct unaligned bytes from maxlen.  */


	/* Compute offsets for vmx loads, and precompute the vbpermq
	   constants for both the 64B and 16B loops.  */
	li	r6,0
	vspltisb  v0,0
	vspltisb  v10,3
	lvsl	  v11,r6,r6
	vslb	  v10,v11,v10

	cmpldi  r4,64		/* Check maxlen < 64.  */
	blt	L(smaller)	/* If maxlen < 64 */

	/* In order to begin the 64B loop, it needs to be 64
	   bytes aligned. So read quadwords until it is aligned or found null
	   bytes. At worst case it will be aligned after the fourth iteration,
	   so unroll the loop to avoid counter checking.  */
	andi.   r7,r5,63		/* Check if is 64 bytes aligned.  */
	beq     cr0,L(preloop_64B)	/* If it is already 64B aligned.  */
	lvx     v1,r5,r6
	vcmpequb.       v1,v1,v0
	addi    r5,r5,16
	addi    r4,r4,-16		/* Decrement maxlen in 16 bytes. */
	bne     cr6,L(found_aligning64B) /* If found null bytes.  */

	/* Unroll 2x above code block until aligned or find null bytes.  */
	andi.   r7,r5,63
	beq     cr0,L(preloop_64B)
	lvx     v1,r5,r6
	vcmpequb.      v1,v1,v0
	addi    r5,r5,16
	addi    r4,r4,-16
	bne     cr6,L(found_aligning64B)

	andi.   r7,r5,63
	beq     cr0,L(preloop_64B)
	lvx     v1,r5,r6
	vcmpequb.      v1,v1,v0
	addi    r5,r5,16
	addi    r4,r4,-16
	bne     cr6,L(found_aligning64B)

	/* At this point it should be 16 bytes aligned.
	   Prepare for the 64B loop.  */
	.p2align 4
L(preloop_64B):
	/* Chec