Skip site navigation (1)Skip section navigation (2)
Date:      Thu, 27 Sep 2018 17:08:30 +0000 (UTC)
From:      Mateusz Guzik <mjg@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r338973 - head/lib/libc/amd64/string
Message-ID:  <201809271708.w8RH8UeV027799@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: mjg
Date: Thu Sep 27 17:08:29 2018
New Revision: 338973
URL: https://svnweb.freebsd.org/changeset/base/338973

Log:
  amd64: reimplement libc memcmp and bcmp with kernel memcmp
  
  Both are significantly slower than hand-coded loops. See r338963 for
  kernel commit.
  
  bcmp differs from memcmp by always returning 1 when a difference is
  found, as opposed to going for a value bigger or lower than 0
  depending on what it is. This means it can do less work. For now the
  code is duplicated and modified. This will get deduplicated after
  another round of optimization when memcmp will get a longer-term form.
  
  Both tested with the glibc suite. While the suite does not have a test
  for bcmp, I created a wrapper routine which verified that values match
  (0 vs 0, 1 vs non-zero).
  
  Reviewed by:	kib
  Approved by:	re (gjb)
  Sponsored by:	The FreeBSD Foundation
  Differential Revision:	https://reviews.freebsd.org/D17336

Modified:
  head/lib/libc/amd64/string/bcmp.S
  head/lib/libc/amd64/string/memcmp.S

Modified: head/lib/libc/amd64/string/bcmp.S
==============================================================================
--- head/lib/libc/amd64/string/bcmp.S	Thu Sep 27 16:43:41 2018	(r338972)
+++ head/lib/libc/amd64/string/bcmp.S	Thu Sep 27 17:08:29 2018	(r338973)
@@ -1,27 +1,121 @@
+/*-
+ * Copyright (c) 2018 The FreeBSD Foundation
+ *
+ * This software was developed by Mateusz Guzik <mjg@FreeBSD.org>
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
 #include <machine/asm.h>
 __FBSDID("$FreeBSD$");
 
-#if 0
-	RCSID("$NetBSD: bcmp.S,v 1.1 2001/06/19 00:25:04 fvdl Exp $")
-#endif
-
 ENTRY(bcmp)
-	cld				/* set compare direction forward */
+	cmpq	$16,%rdx
+	jae	5f
+1:
+	testq	%rdx,%rdx
+	je	3f
+	xorl	%ecx,%ecx
+2:
+	movzbl	(%rdi,%rcx,1),%eax
+	movzbl	(%rsi,%rcx,1),%r8d
+	cmpb	%r8b,%al
+	jne	4f
+	addq	$1,%rcx
+	cmpq	%rcx,%rdx
+	jz	3f
+	movzbl	(%rdi,%rcx,1),%eax
+	movzbl	(%rsi,%rcx,1),%r8d
+	cmpb	%r8b,%al
+	jne	4f
+	addq	$1,%rcx
+	cmpq	%rcx,%rdx
+	jz	3f
+	movzbl	(%rdi,%rcx,1),%eax
+	movzbl	(%rsi,%rcx,1),%r8d
+	cmpb	%r8b,%al
+	jne	4f
+	addq	$1,%rcx
+	cmpq	%rcx,%rdx
+	jz	3f
+	movzbl	(%rdi,%rcx,1),%eax
+	movzbl	(%rsi,%rcx,1),%r8d
+	cmpb	%r8b,%al
+	jne	4f
+	addq	$1,%rcx
+	cmpq	%rcx,%rdx
+	jne	2b
+3:
+	xorl	%eax,%eax
+	ret
+4:
+	movl	$1,%eax
+	ret
+5:
+	cmpq	$32,%rdx
+	jae	7f
+6:
+	/*
+	 * 8 bytes
+	 */
+	movq	(%rdi),%r8
+	movq	(%rsi),%r9
+	cmpq	%r8,%r9
+	jne	4b
+	leaq	8(%rdi),%rdi
+	leaq	8(%rsi),%rsi
+	subq	$8,%rdx
+	cmpq	$8,%rdx
+	jae	6b
+	jl	1b
+	jmp	3b
+7:
+	/*
+	 * 32 bytes
+	 */
+	movq	(%rsi),%r8
+	movq	8(%rsi),%r9
+	subq	(%rdi),%r8
+	subq	8(%rdi),%r9
+	or	%r8,%r9
+	jnz	4b
 
-	movq	%rdx,%rcx		/* compare by words */
-	shrq	$3,%rcx
-	repe
-	cmpsq
-	jne	L1
+	movq	16(%rsi),%r8
+	movq	24(%rsi),%r9
+	subq	16(%rdi),%r8
+	subq	24(%rdi),%r9
+	or	%r8,%r9
+	jnz	4b
 
-	movq	%rdx,%rcx		/* compare remainder by bytes */
-	andq	$7,%rcx
-	repe
-	cmpsb
-L1:
-	setne	%al
-	movsbl	%al,%eax
-	ret
+	leaq	32(%rdi),%rdi
+	leaq	32(%rsi),%rsi
+	subq	$32,%rdx
+	cmpq	$32,%rdx
+	jae	7b
+	jnz	1b
+	jmp	3b
 END(bcmp)
 
 	.section .note.GNU-stack,"",%progbits

Modified: head/lib/libc/amd64/string/memcmp.S
==============================================================================
--- head/lib/libc/amd64/string/memcmp.S	Thu Sep 27 16:43:41 2018	(r338972)
+++ head/lib/libc/amd64/string/memcmp.S	Thu Sep 27 17:08:29 2018	(r338973)
@@ -1,44 +1,121 @@
-/*
- * Written by J.T. Conklin <jtc@NetBSD.org>.
- * Public domain.
- * Adapted for NetBSD/x86_64 by Frank van der Linden <fvdl@wasabisystems.com>
+/*-
+ * Copyright (c) 2018 The FreeBSD Foundation
+ *
+ * This software was developed by Mateusz Guzik <mjg@FreeBSD.org>
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
  */
 
 #include <machine/asm.h>
 __FBSDID("$FreeBSD$");
 
-#if 0
-	RCSID("$NetBSD: memcmp.S,v 1.2 2003/07/26 19:24:39 salo Exp $")
-#endif
-
 ENTRY(memcmp)
-	cld				/* set compare direction forward */
-	movq	%rdx,%rcx		/* compare by longs */
-	shrq	$3,%rcx
-	repe
-	cmpsq
-	jne	L5			/* do we match so far? */
-
-	movq	%rdx,%rcx		/* compare remainder by bytes */
-	andq	$7,%rcx
-	repe
-	cmpsb
-	jne	L6			/* do we match? */
-
-	xorl	%eax,%eax		/* we match, return zero	*/
+	cmpq	$16,%rdx
+	jae	5f
+1:
+	testq	%rdx,%rdx
+	je	3f
+	xorl	%ecx,%ecx
+2:
+	movzbl	(%rdi,%rcx,1),%eax
+	movzbl	(%rsi,%rcx,1),%r8d
+	cmpb	%r8b,%al
+	jne	4f
+	addq	$1,%rcx
+	cmpq	%rcx,%rdx
+	jz	3f
+	movzbl	(%rdi,%rcx,1),%eax
+	movzbl	(%rsi,%rcx,1),%r8d
+	cmpb	%r8b,%al
+	jne	4f
+	addq	$1,%rcx
+	cmpq	%rcx,%rdx
+	jz	3f
+	movzbl	(%rdi,%rcx,1),%eax
+	movzbl	(%rsi,%rcx,1),%r8d
+	cmpb	%r8b,%al
+	jne	4f
+	addq	$1,%rcx
+	cmpq	%rcx,%rdx
+	jz	3f
+	movzbl	(%rdi,%rcx,1),%eax
+	movzbl	(%rsi,%rcx,1),%r8d
+	cmpb	%r8b,%al
+	jne	4f
+	addq	$1,%rcx
+	cmpq	%rcx,%rdx
+	jne	2b
+3:
+	xorl	%eax,%eax
 	ret
-
-L5:	movl	$8,%ecx			/* We know that one of the next	*/
-	subq	%rcx,%rdi		/* eight pairs of bytes do not	*/
-	subq	%rcx,%rsi		/* match.			*/
-	repe
-	cmpsb
-L6:	xorl	%eax,%eax		/* Perform unsigned comparison	*/
-	movb	-1(%rdi),%al
-	xorl	%edx,%edx
-	movb	-1(%rsi),%dl
-	subl    %edx,%eax
+4:
+	subl	%r8d,%eax
 	ret
+5:
+	cmpq	$32,%rdx
+	jae	7f
+6:
+	/*
+	 * 8 bytes
+	 */
+	movq	(%rdi),%r8
+	movq	(%rsi),%r9
+	cmpq	%r8,%r9
+	jne	1b
+	leaq	8(%rdi),%rdi
+	leaq	8(%rsi),%rsi
+	subq	$8,%rdx
+	cmpq	$8,%rdx
+	jae	6b
+	jl	1b
+	jmp	3b
+7:
+	/*
+	 * 32 bytes
+	 */
+	movq	(%rsi),%r8
+	movq	8(%rsi),%r9
+	subq	(%rdi),%r8
+	subq	8(%rdi),%r9
+	or	%r8,%r9
+	jnz	1b
+
+	movq	16(%rsi),%r8
+	movq	24(%rsi),%r9
+	subq	16(%rdi),%r8
+	subq	24(%rdi),%r9
+	or	%r8,%r9
+	jnz	1b
+
+	leaq	32(%rdi),%rdi
+	leaq	32(%rsi),%rsi
+	subq	$32,%rdx
+	cmpq	$32,%rdx
+	jae	7b
+	jnz	1b
+	jmp	3b
 END(memcmp)
 
 	.section .note.GNU-stack,"",%progbits



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201809271708.w8RH8UeV027799>