From owner-svn-src-all@FreeBSD.ORG Fri Nov 20 18:49:20 2009 Return-Path: Delivered-To: svn-src-all@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id E464D1065676; Fri, 20 Nov 2009 18:49:20 +0000 (UTC) (envelope-from jkim@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id D1BBF8FC1F; Fri, 20 Nov 2009 18:49:20 +0000 (UTC) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id nAKInKR5053107; Fri, 20 Nov 2009 18:49:20 GMT (envelope-from jkim@svn.freebsd.org) Received: (from jkim@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id nAKInKt6053100; Fri, 20 Nov 2009 18:49:20 GMT (envelope-from jkim@svn.freebsd.org) Message-Id: <200911201849.nAKInKt6053100@svn.freebsd.org> From: Jung-uk Kim Date: Fri, 20 Nov 2009 18:49:20 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r199603 - in head/sys: amd64/amd64 i386/i386 net X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 20 Nov 2009 18:49:21 -0000 Author: jkim Date: Fri Nov 20 18:49:20 2009 New Revision: 199603 URL: http://svn.freebsd.org/changeset/base/199603 Log: - Allocate scratch memory on stack instead of pre-allocating it with the filter as we do from bpf_filter()[1]. - Revert experimental use of contigmalloc(9)/contigfree(9). It has no performance benefit over malloc(9)/free(9)[2]. Requested by: rwatson[1] Pointed out by: rwatson, jhb, alc[2] Modified: head/sys/amd64/amd64/bpf_jit_machdep.c head/sys/amd64/amd64/bpf_jit_machdep.h head/sys/i386/i386/bpf_jit_machdep.c head/sys/i386/i386/bpf_jit_machdep.h head/sys/net/bpf_jitter.c head/sys/net/bpf_jitter.h Modified: head/sys/amd64/amd64/bpf_jit_machdep.c ============================================================================== --- head/sys/amd64/amd64/bpf_jit_machdep.c Fri Nov 20 17:59:50 2009 (r199602) +++ head/sys/amd64/amd64/bpf_jit_machdep.c Fri Nov 20 18:49:20 2009 (r199603) @@ -53,7 +53,7 @@ __FBSDID("$FreeBSD$"); #include -bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *, int *); +bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *); /* * emit routine to update the jump table @@ -97,7 +97,7 @@ emit_code(bpf_bin_stream *stream, u_int * Function that does the real stuff */ bpf_filter_func -bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem) +bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size) { bpf_bin_stream stream; struct bpf_insn *ins; @@ -111,10 +111,9 @@ bpf_jit_compile(struct bpf_insn *prog, u /* Allocate the reference table for the jumps */ #ifdef _KERNEL - stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int), - M_BPFJIT, M_NOWAIT); + stream.refs = malloc((nins + 1) * sizeof(u_int), M_BPFJIT, M_NOWAIT); #else - stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int)); + stream.refs = malloc((nins + 1) * sizeof(u_int)); #endif if (stream.refs == NULL) return (NULL); @@ -137,8 +136,10 @@ bpf_jit_compile(struct bpf_insn *prog, u ins = prog; /* create the procedure header */ - MOVrq2(RBX, R8); - MOVrq(RDI, RBX); + PUSH(RBP); + MOVrq(RSP, RBP); + SUBib(BPF_MEMWORDS * sizeof(uint32_t), RSP); + MOVrq2(RDI, R8); MOVrd2(ESI, R9D); MOVrd(EDX, EDI); @@ -155,13 +156,11 @@ bpf_jit_compile(struct bpf_insn *prog, u case BPF_RET|BPF_K: MOVid(ins->k, EAX); - MOVrq3(R8, RBX); - RET(); + LEAVE_RET(); break; case BPF_RET|BPF_A: - MOVrq3(R8, RBX); - RET(); + LEAVE_RET(); break; case BPF_LD|BPF_W|BPF_ABS: @@ -171,11 +170,11 @@ bpf_jit_compile(struct bpf_insn *prog, u MOVrd(EDI, ECX); SUBrd(ESI, ECX); CMPid(sizeof(int32_t), ECX); - JAEb(6); + JAEb(4); ZEROrd(EAX); - MOVrq3(R8, RBX); - RET(); - MOVobd(RBX, RSI, EAX); + LEAVE_RET(); + MOVrq3(R8, RCX); + MOVobd(RCX, RSI, EAX); BSWAP(EAX); break; @@ -187,10 +186,10 @@ bpf_jit_compile(struct bpf_insn *prog, u MOVrd(EDI, ECX); SUBrd(ESI, ECX); CMPid(sizeof(int16_t), ECX); - JAEb(4); - MOVrq3(R8, RBX); - RET(); - MOVobw(RBX, RSI, AX); + JAEb(2); + LEAVE_RET(); + MOVrq3(R8, RCX); + MOVobw(RCX, RSI, AX); SWAP_AX(); break; @@ -198,10 +197,10 @@ bpf_jit_compile(struct bpf_insn *prog, u ZEROrd(EAX); MOVid(ins->k, ESI); CMPrd(EDI, ESI); - JBb(4); - MOVrq3(R8, RBX); - RET(); - MOVobb(RBX, RSI, AL); + JBb(2); + LEAVE_RET(); + MOVrq3(R8, RCX); + MOVobb(RCX, RSI, AL); break; case BPF_LD|BPF_W|BPF_LEN: @@ -224,11 +223,11 @@ bpf_jit_compile(struct bpf_insn *prog, u MOVrd(EDI, ECX); SUBrd(ESI, ECX); CMPid(sizeof(int32_t), ECX); - JAEb(6); + JAEb(4); ZEROrd(EAX); - MOVrq3(R8, RBX); - RET(); - MOVobd(RBX, RSI, EAX); + LEAVE_RET(); + MOVrq3(R8, RCX); + MOVobd(RCX, RSI, EAX); BSWAP(EAX); break; @@ -245,10 +244,10 @@ bpf_jit_compile(struct bpf_insn *prog, u MOVrd(EDI, ECX); SUBrd(ESI, ECX); CMPid(sizeof(int16_t), ECX); - JAEb(4); - MOVrq3(R8, RBX); - RET(); - MOVobw(RBX, RSI, AX); + JAEb(2); + LEAVE_RET(); + MOVrq3(R8, RCX); + MOVobw(RCX, RSI, AX); SWAP_AX(); break; @@ -260,22 +259,22 @@ bpf_jit_compile(struct bpf_insn *prog, u MOVrd(EDI, ECX); SUBrd(EDX, ECX); CMPrd(ESI, ECX); - JAb(4); - MOVrq3(R8, RBX); - RET(); + JAb(2); + LEAVE_RET(); + MOVrq3(R8, RCX); ADDrd(EDX, ESI); - MOVobb(RBX, RSI, AL); + MOVobb(RCX, RSI, AL); break; case BPF_LDX|BPF_MSH|BPF_B: MOVid(ins->k, ESI); CMPrd(EDI, ESI); - JBb(6); + JBb(4); ZEROrd(EAX); - MOVrq3(R8, RBX); - RET(); + LEAVE_RET(); ZEROrd(EDX); - MOVobb(RBX, RSI, DL); + MOVrq3(R8, RCX); + MOVobb(RCX, RSI, DL); ANDib(0x0f, DL); SHLib(2, EDX); break; @@ -289,15 +288,13 @@ bpf_jit_compile(struct bpf_insn *prog, u break; case BPF_LD|BPF_MEM: - MOViq((uintptr_t)mem, RCX); - MOVid(ins->k * 4, ESI); - MOVobd(RCX, RSI, EAX); + MOVid(ins->k * sizeof(uint32_t), ESI); + MOVobd(RSP, RSI, EAX); break; case BPF_LDX|BPF_MEM: - MOViq((uintptr_t)mem, RCX); - MOVid(ins->k * 4, ESI); - MOVobd(RCX, RSI, EDX); + MOVid(ins->k * sizeof(uint32_t), ESI); + MOVobd(RSP, RSI, EDX); break; case BPF_ST: @@ -306,15 +303,13 @@ bpf_jit_compile(struct bpf_insn *prog, u * be optimized if the previous instruction * was already of this type */ - MOViq((uintptr_t)mem, RCX); - MOVid(ins->k * 4, ESI); - MOVomd(EAX, RCX, RSI); + MOVid(ins->k * sizeof(uint32_t), ESI); + MOVomd(EAX, RSP, RSI); break; case BPF_STX: - MOViq((uintptr_t)mem, RCX); - MOVid(ins->k * 4, ESI); - MOVomd(EDX, RCX, RSI); + MOVid(ins->k * sizeof(uint32_t), ESI); + MOVomd(EDX, RSP, RSI); break; case BPF_JMP|BPF_JA: @@ -394,10 +389,9 @@ bpf_jit_compile(struct bpf_insn *prog, u case BPF_ALU|BPF_DIV|BPF_X: TESTrd(EDX, EDX); - JNEb(6); + JNEb(4); ZEROrd(EAX); - MOVrq3(R8, RBX); - RET(); + LEAVE_RET(); MOVrd(EDX, ECX); ZEROrd(EDX); DIVrd(ECX); @@ -490,13 +484,12 @@ bpf_jit_compile(struct bpf_insn *prog, u } #ifdef _KERNEL - stream.ibuf = (char *)contigmalloc(stream.cur_ip, M_BPFJIT, - M_NOWAIT, 0, ~0UL, 16, 0); + stream.ibuf = malloc(stream.cur_ip, M_BPFJIT, M_NOWAIT); if (stream.ibuf == NULL) break; #else - stream.ibuf = (char *)mmap(NULL, stream.cur_ip, - PROT_READ | PROT_WRITE, MAP_ANON, -1, 0); + stream.ibuf = mmap(NULL, stream.cur_ip, PROT_READ | PROT_WRITE, + MAP_ANON, -1, 0); if (stream.ibuf == MAP_FAILED) { stream.ibuf = NULL; break; Modified: head/sys/amd64/amd64/bpf_jit_machdep.h ============================================================================== --- head/sys/amd64/amd64/bpf_jit_machdep.h Fri Nov 20 17:59:50 2009 (r199602) +++ head/sys/amd64/amd64/bpf_jit_machdep.h Fri Nov 20 18:49:20 2009 (r199603) @@ -215,9 +215,14 @@ typedef void (*emit_func)(bpf_bin_stream emitm(&stream, 0xc486, 2); \ } while (0) -/* ret */ -#define RET() do { \ - emitm(&stream, 0xc3, 1); \ +/* pushq r64 */ +#define PUSH(r64) do { \ + emitm(&stream, (5 << 4) | (0 << 3) | (r64 & 0x7), 1); \ +} while (0) + +/* leave/ret */ +#define LEAVE_RET() do { \ + emitm(&stream, 0xc3c9, 2); \ } while (0) /* addl sr32,dr32 */ @@ -253,6 +258,13 @@ typedef void (*emit_func)(bpf_bin_stream emitm(&stream, i32, 4); \ } while (0) +/* subq i8,r64 */ +#define SUBib(i8, r64) do { \ + emitm(&stream, 0x8348, 2); \ + emitm(&stream, (29 << 3) | (r64 & 0x7), 1); \ + emitm(&stream, i8, 1); \ +} while (0) + /* mull r32 */ #define MULrd(r32) do { \ emitm(&stream, 0xf7, 1); \ Modified: head/sys/i386/i386/bpf_jit_machdep.c ============================================================================== --- head/sys/i386/i386/bpf_jit_machdep.c Fri Nov 20 17:59:50 2009 (r199602) +++ head/sys/i386/i386/bpf_jit_machdep.c Fri Nov 20 18:49:20 2009 (r199603) @@ -53,7 +53,7 @@ __FBSDID("$FreeBSD$"); #include -bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *, int *); +bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *); /* * emit routine to update the jump table @@ -97,7 +97,7 @@ emit_code(bpf_bin_stream *stream, u_int * Function that does the real stuff */ bpf_filter_func -bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem) +bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size) { bpf_bin_stream stream; struct bpf_insn *ins; @@ -111,10 +111,9 @@ bpf_jit_compile(struct bpf_insn *prog, u /* Allocate the reference table for the jumps */ #ifdef _KERNEL - stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int), - M_BPFJIT, M_NOWAIT); + stream.refs = malloc((nins + 1) * sizeof(u_int), M_BPFJIT, M_NOWAIT); #else - stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int)); + stream.refs = malloc((nins + 1) * sizeof(u_int)); #endif if (stream.refs == NULL) return (NULL); @@ -139,6 +138,7 @@ bpf_jit_compile(struct bpf_insn *prog, u /* create the procedure header */ PUSH(EBP); MOVrd(ESP, EBP); + SUBib(BPF_MEMWORDS * sizeof(uint32_t), ESP); PUSH(EDI); PUSH(ESI); PUSH(EBX); @@ -310,14 +310,16 @@ bpf_jit_compile(struct bpf_insn *prog, u break; case BPF_LD|BPF_MEM: - MOVid((uintptr_t)mem, ECX); - MOVid(ins->k * 4, ESI); + MOVrd(EBP, ECX); + MOVid(((int)ins->k - BPF_MEMWORDS) * + sizeof(uint32_t), ESI); MOVobd(ECX, ESI, EAX); break; case BPF_LDX|BPF_MEM: - MOVid((uintptr_t)mem, ECX); - MOVid(ins->k * 4, ESI); + MOVrd(EBP, ECX); + MOVid(((int)ins->k - BPF_MEMWORDS) * + sizeof(uint32_t), ESI); MOVobd(ECX, ESI, EDX); break; @@ -327,14 +329,16 @@ bpf_jit_compile(struct bpf_insn *prog, u * be optimized if the previous instruction * was already of this type */ - MOVid((uintptr_t)mem, ECX); - MOVid(ins->k * 4, ESI); + MOVrd(EBP, ECX); + MOVid(((int)ins->k - BPF_MEMWORDS) * + sizeof(uint32_t), ESI); MOVomd(EAX, ECX, ESI); break; case BPF_STX: - MOVid((uintptr_t)mem, ECX); - MOVid(ins->k * 4, ESI); + MOVrd(EBP, ECX); + MOVid(((int)ins->k - BPF_MEMWORDS) * + sizeof(uint32_t), ESI); MOVomd(EDX, ECX, ESI); break; @@ -513,13 +517,12 @@ bpf_jit_compile(struct bpf_insn *prog, u } #ifdef _KERNEL - stream.ibuf = (char *)contigmalloc(stream.cur_ip, M_BPFJIT, - M_NOWAIT, 0, ~0UL, 16, 0); + stream.ibuf = malloc(stream.cur_ip, M_BPFJIT, M_NOWAIT); if (stream.ibuf == NULL) break; #else - stream.ibuf = (char *)mmap(NULL, stream.cur_ip, - PROT_READ | PROT_WRITE, MAP_ANON, -1, 0); + stream.ibuf = mmap(NULL, stream.cur_ip, PROT_READ | PROT_WRITE, + MAP_ANON, -1, 0); if (stream.ibuf == MAP_FAILED) { stream.ibuf = NULL; break; Modified: head/sys/i386/i386/bpf_jit_machdep.h ============================================================================== --- head/sys/i386/i386/bpf_jit_machdep.h Fri Nov 20 17:59:50 2009 (r199602) +++ head/sys/i386/i386/bpf_jit_machdep.h Fri Nov 20 18:49:20 2009 (r199603) @@ -203,6 +203,13 @@ typedef void (*emit_func)(bpf_bin_stream emitm(&stream, i32, 4); \ } while (0) +/* subl i8,r32 */ +#define SUBib(i8, r32) do { \ + emitm(&stream, 0x83, 1); \ + emitm(&stream, (29 << 3) | (r32 & 0x7), 1); \ + emitm(&stream, i8, 1); \ +} while (0) + /* mull r32 */ #define MULrd(r32) do { \ emitm(&stream, 0xf7, 1); \ Modified: head/sys/net/bpf_jitter.c ============================================================================== --- head/sys/net/bpf_jitter.c Fri Nov 20 17:59:50 2009 (r199602) +++ head/sys/net/bpf_jitter.c Fri Nov 20 18:49:20 2009 (r199603) @@ -51,7 +51,7 @@ __FBSDID("$FreeBSD$"); #include #include -bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *, int *); +bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *); static u_int bpf_jit_accept_all(u_char *, u_int, u_int); @@ -70,7 +70,7 @@ bpf_jitter(struct bpf_insn *fp, int nins /* Allocate the filter structure */ filter = (struct bpf_jit_filter *)malloc(sizeof(*filter), - M_BPFJIT, M_NOWAIT | M_ZERO); + M_BPFJIT, M_NOWAIT); if (filter == NULL) return (NULL); @@ -81,8 +81,7 @@ bpf_jitter(struct bpf_insn *fp, int nins } /* Create the binary */ - if ((filter->func = bpf_jit_compile(fp, nins, &filter->size, - filter->mem)) == NULL) { + if ((filter->func = bpf_jit_compile(fp, nins, &filter->size)) == NULL) { free(filter, M_BPFJIT); return (NULL); } @@ -95,7 +94,7 @@ bpf_destroy_jit_filter(bpf_jit_filter *f { if (filter->func != bpf_jit_accept_all) - contigfree(filter->func, filter->size, M_BPFJIT); + free(filter->func, M_BPFJIT); free(filter, M_BPFJIT); } #else @@ -108,7 +107,6 @@ bpf_jitter(struct bpf_insn *fp, int nins filter = (struct bpf_jit_filter *)malloc(sizeof(*filter)); if (filter == NULL) return (NULL); - memset(filter, 0, sizeof(*filter)); /* No filter means accept all */ if (fp == NULL || nins == 0) { @@ -117,8 +115,7 @@ bpf_jitter(struct bpf_insn *fp, int nins } /* Create the binary */ - if ((filter->func = bpf_jit_compile(fp, nins, &filter->size, - filter->mem)) == NULL) { + if ((filter->func = bpf_jit_compile(fp, nins, &filter->size)) == NULL) { free(filter); return (NULL); } Modified: head/sys/net/bpf_jitter.h ============================================================================== --- head/sys/net/bpf_jitter.h Fri Nov 20 17:59:50 2009 (r199602) +++ head/sys/net/bpf_jitter.h Fri Nov 20 18:49:20 2009 (r199603) @@ -54,7 +54,6 @@ typedef struct bpf_jit_filter { /* The native filtering binary, in the form of a bpf_filter_func. */ bpf_filter_func func; size_t size; - int mem[BPF_MEMWORDS]; /* Scratch memory */ } bpf_jit_filter; /*