Skip site navigation (1)Skip section navigation (2)
Date:      Thu, 17 May 2018 15:24:53 +0000 (UTC)
From:      Konstantin Belousov <kib@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-11@freebsd.org
Subject:   svn commit: r333720 - in stable/11/sys: amd64/amd64 i386/i386 kern sys
Message-ID:  <201805171524.w4HFOr90093633@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: kib
Date: Thu May 17 15:24:53 2018
New Revision: 333720
URL: https://svnweb.freebsd.org/changeset/base/333720

Log:
  MFC r333228
  Implement support for ifuncs in the kernel linker on x86.
  
  MFC r333411:
  Avoid calls to bzero() before ireloc
  
  Approved by:	re (marius)

Modified:
  stable/11/sys/amd64/amd64/elf_machdep.c
  stable/11/sys/amd64/amd64/machdep.c
  stable/11/sys/i386/i386/elf_machdep.c
  stable/11/sys/i386/i386/machdep.c
  stable/11/sys/kern/link_elf.c
  stable/11/sys/kern/link_elf_obj.c
  stable/11/sys/sys/linker.h
Directory Properties:
  stable/11/   (props changed)

Modified: stable/11/sys/amd64/amd64/elf_machdep.c
==============================================================================
--- stable/11/sys/amd64/amd64/elf_machdep.c	Thu May 17 15:21:19 2018	(r333719)
+++ stable/11/sys/amd64/amd64/elf_machdep.c	Thu May 17 15:24:53 2018	(r333720)
@@ -173,10 +173,13 @@ elf64_dump_thread(struct thread *td, void *dst, size_t
 	*off = len;
 }
 
+#define	ERI_LOCAL	0x0001
+#define	ERI_ONLYIFUNC	0x0002
+
 /* Process one elf relocation with addend. */
 static int
 elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
-    int type, int local, elf_lookup_fn lookup)
+    int type, elf_lookup_fn lookup, int flags)
 {
 	Elf64_Addr *where, val;
 	Elf32_Addr *where32, val32;
@@ -215,6 +218,9 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbas
 		panic("unknown reloc type %d\n", type);
 	}
 
+	if (((flags & ERI_ONLYIFUNC) == 0) ^ (rtype != R_X86_64_IRELATIVE))
+		return (0);
+
 	switch (rtype) {
 		case R_X86_64_NONE:	/* none */
 			break;
@@ -273,6 +279,13 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbas
 				*where = val;
 			break;
 
+		case R_X86_64_IRELATIVE:
+			addr = relocbase + addend;
+			val = ((Elf64_Addr (*)(void))addr)();
+			if (*where != val)
+				*where = val;
+			break;
+
 		default:
 			printf("kldload: unexpected relocation type %ld\n",
 			       rtype);
@@ -282,11 +295,20 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbas
 }
 
 int
+elf_reloc_ifunc(linker_file_t lf, Elf_Addr relocbase, const void *data,
+    int type, elf_lookup_fn lookup)
+{
+
+	return (elf_reloc_internal(lf, relocbase, data, type, lookup,
+	    ERI_ONLYIFUNC));
+}
+
+int
 elf_reloc(linker_file_t lf, Elf_Addr relocbase, const void *data, int type,
     elf_lookup_fn lookup)
 {
 
-	return (elf_reloc_internal(lf, relocbase, data, type, 0, lookup));
+	return (elf_reloc_internal(lf, relocbase, data, type, lookup, 0));
 }
 
 int
@@ -294,7 +316,8 @@ elf_reloc_local(linker_file_t lf, Elf_Addr relocbase, 
     int type, elf_lookup_fn lookup)
 {
 
-	return (elf_reloc_internal(lf, relocbase, data, type, 1, lookup));
+	return (elf_reloc_internal(lf, relocbase, data, type, lookup,
+	    ERI_LOCAL));
 }
 
 int

Modified: stable/11/sys/amd64/amd64/machdep.c
==============================================================================
--- stable/11/sys/amd64/amd64/machdep.c	Thu May 17 15:21:19 2018	(r333719)
+++ stable/11/sys/amd64/amd64/machdep.c	Thu May 17 15:24:53 2018	(r333720)
@@ -1564,16 +1564,23 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 	size_t kstack0_sz;
 	int late_console;
 
-	/*
- 	 * This may be done better later if it gets more high level
- 	 * components in it. If so just link td->td_proc here.
-	 */
-	proc_linkup0(&proc0, &thread0);
-
 	kmdp = init_ops.parse_preload_data(modulep);
 
 	identify_cpu1();
 	identify_hypervisor();
+	/*
+	 * hw.cpu_stdext_disable is ignored by the call, it will be
+	 * re-evaluted by the below call to finishidentcpu().
+	 */
+	identify_cpu2();
+
+	link_elf_ireloc(kmdp);
+
+	/*
+	 * This may be done better later if it gets more high level
+	 * components in it. If so just link td->td_proc here.
+	 */
+	proc_linkup0(&proc0, &thread0);
 
 	/* Init basic tunables, hz etc */
 	init_param1();

Modified: stable/11/sys/i386/i386/elf_machdep.c
==============================================================================
--- stable/11/sys/i386/i386/elf_machdep.c	Thu May 17 15:21:19 2018	(r333719)
+++ stable/11/sys/i386/i386/elf_machdep.c	Thu May 17 15:24:53 2018	(r333720)
@@ -158,10 +158,13 @@ elf32_dump_thread(struct thread *td, void *dst, size_t
 	*off = len;
 }
 
+#define	ERI_LOCAL	0x0001
+#define	ERI_ONLYIFUNC	0x0002
+
 /* Process one elf relocation with addend. */
 static int
 elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
-    int type, int local, elf_lookup_fn lookup)
+    int type, elf_lookup_fn lookup, int flags)
 {
 	Elf_Addr *where;
 	Elf_Addr addr;
@@ -190,7 +193,10 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbas
 		panic("unknown reloc type %d\n", type);
 	}
 
-	if (local) {
+	if (((flags & ERI_ONLYIFUNC) == 0) ^ (rtype != R_386_IRELATIVE))
+		return (0);
+
+	if ((flags & ERI_LOCAL) != 0) {
 		if (rtype == R_386_RELATIVE) {	/* A + B */
 			addr = elf_relocaddr(lf, relocbase + addend);
 			if (*where != addr)
@@ -242,6 +248,12 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbas
 		case R_386_RELATIVE:
 			break;
 
+		case R_386_IRELATIVE:
+			addr = relocbase + addend;
+			addr = ((Elf_Addr (*)(void))addr)();
+			if (*where != addr)
+				*where = addr;
+			break;
 		default:
 			printf("kldload: unexpected relocation type %d\n",
 			       rtype);
@@ -251,11 +263,20 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbas
 }
 
 int
+elf_reloc_ifunc(linker_file_t lf, Elf_Addr relocbase, const void *data,
+    int type, elf_lookup_fn lookup)
+{
+
+	return (elf_reloc_internal(lf, relocbase, data, type, lookup,
+	    ERI_ONLYIFUNC));
+}
+
+int
 elf_reloc(linker_file_t lf, Elf_Addr relocbase, const void *data, int type,
     elf_lookup_fn lookup)
 {
 
-	return (elf_reloc_internal(lf, relocbase, data, type, 0, lookup));
+	return (elf_reloc_internal(lf, relocbase, data, type, lookup, 0));
 }
 
 int
@@ -263,7 +284,8 @@ elf_reloc_local(linker_file_t lf, Elf_Addr relocbase, 
     int type, elf_lookup_fn lookup)
 {
 
-	return (elf_reloc_internal(lf, relocbase, data, type, 1, lookup));
+	return (elf_reloc_internal(lf, relocbase, data, type, lookup,
+	    ERI_LOCAL));
 }
 
 int

Modified: stable/11/sys/i386/i386/machdep.c
==============================================================================
--- stable/11/sys/i386/i386/machdep.c	Thu May 17 15:21:19 2018	(r333719)
+++ stable/11/sys/i386/i386/machdep.c	Thu May 17 15:24:53 2018	(r333720)
@@ -2441,6 +2441,7 @@ init386(int first)
 	int gsel_tss, metadata_missing, x, pa;
 	struct pcpu *pc;
 	struct xstate_hdr *xhdr;
+	caddr_t kmdp;
 	int late_console;
 
 	thread0.td_kstack = proc0kstack;
@@ -2687,6 +2688,9 @@ init386(int first)
 		cninit();
 		i386_kdb_init();
 	}
+
+	kmdp = preload_search_by_type("elf kernel");
+	link_elf_ireloc(kmdp);
 
 	vm86_initialize();
 	getmemsize(first);

Modified: stable/11/sys/kern/link_elf.c
==============================================================================
--- stable/11/sys/kern/link_elf.c	Thu May 17 15:21:19 2018	(r333719)
+++ stable/11/sys/kern/link_elf.c	Thu May 17 15:24:53 2018	(r333720)
@@ -188,6 +188,9 @@ static struct linker_class link_elf_class = {
 
 static int	parse_dynamic(elf_file_t);
 static int	relocate_file(elf_file_t);
+static int	relocate_file1(elf_file_t ef, int (*elf_reloc_func)(
+		    linker_file_t lf, Elf_Addr relocbase, const void *data,
+		    int type, elf_lookup_fn lookup));
 static int	link_elf_preload_parse_symbols(elf_file_t);
 
 static struct elf_set_head set_pcpu_list;
@@ -1183,7 +1186,8 @@ symbol_name(elf_file_t ef, Elf_Size r_info)
 }
 
 static int
-relocate_file(elf_file_t ef)
+relocate_file1(elf_file_t ef, int (*elf_reloc_func)(linker_file_t lf,
+    Elf_Addr relocbase, const void *data, int type, elf_lookup_fn lookup))
 {
 	const Elf_Rel *rellim;
 	const Elf_Rel *rel;
@@ -1197,7 +1201,7 @@ relocate_file(elf_file_t ef)
 		rellim = (const Elf_Rel *)
 		    ((const char *)ef->rel + ef->relsize);
 		while (rel < rellim) {
-			if (elf_reloc(&ef->lf, (Elf_Addr)ef->address, rel,
+			if (elf_reloc_func(&ef->lf, (Elf_Addr)ef->address, rel,
 			    ELF_RELOC_REL, elf_lookup)) {
 				symname = symbol_name(ef, rel->r_info);
 				printf("link_elf: symbol %s undefined\n", symname);
@@ -1213,7 +1217,7 @@ relocate_file(elf_file_t ef)
 		relalim = (const Elf_Rela *)
 		    ((const char *)ef->rela + ef->relasize);
 		while (rela < relalim) {
-			if (elf_reloc(&ef->lf, (Elf_Addr)ef->address, rela,
+			if (elf_reloc_func(&ef->lf, (Elf_Addr)ef->address, rela,
 			    ELF_RELOC_RELA, elf_lookup)) {
 				symname = symbol_name(ef, rela->r_info);
 				printf("link_elf: symbol %s undefined\n",
@@ -1230,7 +1234,7 @@ relocate_file(elf_file_t ef)
 		rellim = (const Elf_Rel *)
 		    ((const char *)ef->pltrel + ef->pltrelsize);
 		while (rel < rellim) {
-			if (elf_reloc(&ef->lf, (Elf_Addr)ef->address, rel,
+			if (elf_reloc_func(&ef->lf, (Elf_Addr)ef->address, rel,
 			    ELF_RELOC_REL, elf_lookup)) {
 				symname = symbol_name(ef, rel->r_info);
 				printf("link_elf: symbol %s undefined\n",
@@ -1247,7 +1251,7 @@ relocate_file(elf_file_t ef)
 		relalim = (const Elf_Rela *)
 		    ((const char *)ef->pltrela + ef->pltrelasize);
 		while (rela < relalim) {
-			if (elf_reloc(&ef->lf, (Elf_Addr)ef->address, rela,
+			if (elf_reloc_func(&ef->lf, (Elf_Addr)ef->address, rela,
 			    ELF_RELOC_RELA, elf_lookup)) {
 				symname = symbol_name(ef, rela->r_info);
 				printf("link_elf: symbol %s undefined\n",
@@ -1261,6 +1265,19 @@ relocate_file(elf_file_t ef)
 	return (0);
 }
 
+static int
+relocate_file(elf_file_t ef)
+{
+	int e;
+
+	e = relocate_file1(ef, elf_reloc);
+#if defined(__i386__) || defined(__amd64__)
+	if (e == 0)
+		e = relocate_file1(ef, elf_reloc_ifunc);
+#endif
+	return (e);
+}
+
 /*
  * Hash function for symbol table lookup.  Don't even think about changing
  * this.  It is specified by the System V ABI.
@@ -1318,7 +1335,8 @@ link_elf_lookup_symbol(linker_file_t lf, const char* n
 		if (strcmp(name, strp) == 0) {
 			if (symp->st_shndx != SHN_UNDEF ||
 			    (symp->st_value != 0 &&
-			     ELF_ST_TYPE(symp->st_info) == STT_FUNC)) {
+			    (ELF_ST_TYPE(symp->st_info) == STT_FUNC ||
+			    ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC))) {
 				*sym = (c_linker_sym_t) symp;
 				return (0);
 			}
@@ -1338,7 +1356,8 @@ link_elf_lookup_symbol(linker_file_t lf, const char* n
 		if (strcmp(name, strp) == 0) {
 			if (symp->st_shndx != SHN_UNDEF ||
 			    (symp->st_value != 0 &&
-			     ELF_ST_TYPE(symp->st_info) == STT_FUNC)) {
+			    (ELF_ST_TYPE(symp->st_info) == STT_FUNC ||
+			    ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC))) {
 				*sym = (c_linker_sym_t) symp;
 				return (0);
 			}
@@ -1353,12 +1372,18 @@ static int
 link_elf_symbol_values(linker_file_t lf, c_linker_sym_t sym,
     linker_symval_t *symval)
 {
-	elf_file_t ef = (elf_file_t) lf;
-	const Elf_Sym* es = (const Elf_Sym*) sym;
+	elf_file_t ef;
+	const Elf_Sym *es;
+	caddr_t val;
 
+	ef = (elf_file_t)lf;
+	es = (const Elf_Sym *)sym;
 	if (es >= ef->symtab && es < (ef->symtab + ef->nchains)) {
 		symval->name = ef->strtab + es->st_name;
-		symval->value = (caddr_t) ef->address + es->st_value;
+		val = (caddr_t)ef->address + es->st_value;
+		if (ELF_ST_TYPE(es->st_info) == STT_GNU_IFUNC)
+			val = ((caddr_t (*)(void))val)();
+		symval->value = val;
 		symval->size = es->st_size;
 		return (0);
 	}
@@ -1366,7 +1391,10 @@ link_elf_symbol_values(linker_file_t lf, c_linker_sym_
 		return (ENOENT);
 	if (es >= ef->ddbsymtab && es < (ef->ddbsymtab + ef->ddbsymcnt)) {
 		symval->name = ef->ddbstrtab + es->st_name;
-		symval->value = (caddr_t) ef->address + es->st_value;
+		val = (caddr_t)ef->address + es->st_value;
+		if (ELF_ST_TYPE(es->st_info) == STT_GNU_IFUNC)
+			val = ((caddr_t (*)(void))val)();
+		symval->value = val;
 		symval->size = es->st_size;
 		return (0);
 	}
@@ -1476,7 +1504,8 @@ link_elf_each_function_name(linker_file_t file,
 	/* Exhaustive search */
 	for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) {
 		if (symp->st_value != 0 &&
-		    ELF_ST_TYPE(symp->st_info) == STT_FUNC) {
+		    (ELF_ST_TYPE(symp->st_info) == STT_FUNC ||
+		    ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC)) {
 			error = callback(ef->ddbstrtab + symp->st_name, opaque);
 			if (error != 0)
 				return (error);
@@ -1497,7 +1526,8 @@ link_elf_each_function_nameval(linker_file_t file,
 	/* Exhaustive search */
 	for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) {
 		if (symp->st_value != 0 &&
-		    ELF_ST_TYPE(symp->st_info) == STT_FUNC) {
+		    (ELF_ST_TYPE(symp->st_info) == STT_FUNC ||
+		    ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC)) {
 			error = link_elf_symbol_values(file,
 			    (c_linker_sym_t) symp, &symval);
 			if (error != 0)
@@ -1656,3 +1686,27 @@ link_elf_strtab_get(linker_file_t lf, caddr_t *strtab)
 
 	return (ef->ddbstrcnt);
 }
+
+#if defined(__i386__) || defined(__amd64__)
+void
+link_elf_ireloc(caddr_t kmdp)
+{
+	struct elf_file eff;
+	elf_file_t ef;
+	volatile char *c;
+	size_t i;
+
+	ef =  &eff;
+
+	/* Do not use bzero/memset before ireloc is done. */
+	for (c = (char *)ef, i = 0; i < sizeof(*ef); i++)
+		c[i] = 0;
+
+	ef->modptr = kmdp;
+	ef->dynamic = (Elf_Dyn *)&_DYNAMIC;
+	parse_dynamic(ef);
+	ef->address = 0;
+	link_elf_preload_parse_symbols(ef);
+	relocate_file1(ef, elf_reloc_ifunc);
+}
+#endif

Modified: stable/11/sys/kern/link_elf_obj.c
==============================================================================
--- stable/11/sys/kern/link_elf_obj.c	Thu May 17 15:21:19 2018	(r333719)
+++ stable/11/sys/kern/link_elf_obj.c	Thu May 17 15:24:53 2018	(r333720)
@@ -1194,12 +1194,19 @@ static int
 link_elf_symbol_values(linker_file_t lf, c_linker_sym_t sym,
     linker_symval_t *symval)
 {
-	elf_file_t ef = (elf_file_t) lf;
-	const Elf_Sym *es = (const Elf_Sym*) sym;
+	elf_file_t ef;
+	const Elf_Sym *es;
+	caddr_t val;
 
+	ef = (elf_file_t) lf;
+	es = (const Elf_Sym*) sym;
+	val = (caddr_t)es->st_value;
 	if (es >= ef->ddbsymtab && es < (ef->ddbsymtab + ef->ddbsymcnt)) {
 		symval->name = ef->ddbstrtab + es->st_name;
-		symval->value = (caddr_t)es->st_value;
+		val = (caddr_t)es->st_value;
+		if (ELF_ST_TYPE(es->st_info) == STT_GNU_IFUNC)
+			val = ((caddr_t (*)(void))val)();
+		symval->value = val;
 		symval->size = es->st_size;
 		return 0;
 	}
@@ -1284,7 +1291,8 @@ link_elf_each_function_name(linker_file_t file,
 	/* Exhaustive search */
 	for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) {
 		if (symp->st_value != 0 &&
-		    ELF_ST_TYPE(symp->st_info) == STT_FUNC) {
+		    (ELF_ST_TYPE(symp->st_info) == STT_FUNC ||
+		    ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC)) {
 			error = callback(ef->ddbstrtab + symp->st_name, opaque);
 			if (error)
 				return (error);
@@ -1305,8 +1313,10 @@ link_elf_each_function_nameval(linker_file_t file,
 	/* Exhaustive search */
 	for (i = 0, symp = ef->ddbsymtab; i < ef->ddbsymcnt; i++, symp++) {
 		if (symp->st_value != 0 &&
-		    ELF_ST_TYPE(symp->st_info) == STT_FUNC) {
-			error = link_elf_symbol_values(file, (c_linker_sym_t) symp, &symval);
+		    (ELF_ST_TYPE(symp->st_info) == STT_FUNC ||
+		    ELF_ST_TYPE(symp->st_info) == STT_GNU_IFUNC)) {
+			error = link_elf_symbol_values(file,
+			    (c_linker_sym_t)symp, &symval);
 			if (error)
 				return (error);
 			error = callback(file, i, &symval, opaque);

Modified: stable/11/sys/sys/linker.h
==============================================================================
--- stable/11/sys/sys/linker.h	Thu May 17 15:21:19 2018	(r333719)
+++ stable/11/sys/sys/linker.h	Thu May 17 15:24:53 2018	(r333720)
@@ -269,11 +269,16 @@ extern int kld_debug;
 typedef int elf_lookup_fn(linker_file_t, Elf_Size, int, Elf_Addr *);
 
 /* Support functions */
-int	elf_reloc(linker_file_t _lf, Elf_Addr base, const void *_rel, int _type, elf_lookup_fn _lu);
-int	elf_reloc_local(linker_file_t _lf, Elf_Addr base, const void *_rel, int _type, elf_lookup_fn _lu);
+int	elf_reloc(linker_file_t _lf, Elf_Addr base, const void *_rel,
+	    int _type, elf_lookup_fn _lu);
+int	elf_reloc_ifunc(linker_file_t _lf, Elf_Addr base, const void *_rel,
+	    int _type, elf_lookup_fn _lu);
+int	elf_reloc_local(linker_file_t _lf, Elf_Addr base, const void *_rel,
+	    int _type, elf_lookup_fn _lu);
 Elf_Addr elf_relocaddr(linker_file_t _lf, Elf_Addr addr);
 const Elf_Sym *elf_get_sym(linker_file_t _lf, Elf_Size _symidx);
 const char *elf_get_symname(linker_file_t _lf, Elf_Size _symidx);
+void	link_elf_ireloc(caddr_t kmdp);
 
 typedef struct linker_ctf {
 	const uint8_t 	*ctftab;	/* Decompressed CTF data. */



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201805171524.w4HFOr90093633>