Date: Sat, 9 Jun 2018 23:45:06 +0000 (UTC) From: Rick Macklem <rmacklem@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r334905 - in projects/pnfs-planb-server/sys: amd64/amd64 amd64/include arm/arm arm/conf arm/xscale/ixp425 arm64/arm64 arm64/conf cam/mmc cddl/compat/opensolaris/kern cddl/contrib/openso... Message-ID: <201806092345.w59Nj6QL002170@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: rmacklem Date: Sat Jun 9 23:45:05 2018 New Revision: 334905 URL: https://svnweb.freebsd.org/changeset/base/334905 Log: Merge in an up to date kernel from current/head. Added: projects/pnfs-planb-server/sys/compat/linuxkpi/common/include/linux/export.h - copied unchanged from r334904, head/sys/compat/linuxkpi/common/include/linux/export.h projects/pnfs-planb-server/sys/modules/tcp/rack/ - copied from r334904, head/sys/modules/tcp/rack/ projects/pnfs-planb-server/sys/netinet/tcp_stacks/rack.c - copied unchanged from r334904, head/sys/netinet/tcp_stacks/rack.c projects/pnfs-planb-server/sys/netinet/tcp_stacks/rack_bbr_common.h - copied unchanged from r334904, head/sys/netinet/tcp_stacks/rack_bbr_common.h projects/pnfs-planb-server/sys/netinet/tcp_stacks/sack_filter.c - copied unchanged from r334904, head/sys/netinet/tcp_stacks/sack_filter.c projects/pnfs-planb-server/sys/netinet/tcp_stacks/sack_filter.h - copied unchanged from r334904, head/sys/netinet/tcp_stacks/sack_filter.h projects/pnfs-planb-server/sys/netinet/tcp_stacks/tcp_rack.h - copied unchanged from r334904, head/sys/netinet/tcp_stacks/tcp_rack.h Deleted: projects/pnfs-planb-server/sys/dev/hwpmc/hwpmc_xscale.c Modified: projects/pnfs-planb-server/sys/amd64/amd64/machdep.c projects/pnfs-planb-server/sys/amd64/amd64/mp_machdep.c projects/pnfs-planb-server/sys/amd64/amd64/pmap.c projects/pnfs-planb-server/sys/amd64/amd64/support.S projects/pnfs-planb-server/sys/amd64/amd64/trap.c projects/pnfs-planb-server/sys/amd64/include/cpufunc.h projects/pnfs-planb-server/sys/amd64/include/pmc_mdep.h projects/pnfs-planb-server/sys/arm/arm/pmu.c projects/pnfs-planb-server/sys/arm/conf/GENERIC-NODEBUG projects/pnfs-planb-server/sys/arm/xscale/ixp425/files.ixp425 projects/pnfs-planb-server/sys/arm64/arm64/swtch.S projects/pnfs-planb-server/sys/arm64/conf/GENERIC-NODEBUG projects/pnfs-planb-server/sys/cam/mmc/mmc_da.c projects/pnfs-planb-server/sys/cddl/compat/opensolaris/kern/opensolaris_taskq.c projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/ddt.c projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_objset.c projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_traverse.c projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_scan.c projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/metaslab.c projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/range_tree.c projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/arc.h projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_pool.h projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_scan.h projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/range_tree.h projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/spa_impl.h projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/vdev.h projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/vdev_impl.h projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_vfsops.h projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio.h projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_disk.c projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_file.c projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_geom.c projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_indirect.c projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_mirror.c projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_missing.c projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_queue.c projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_raidz.c projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_root.c projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zap.c projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_dir.c projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/sys/fs/zfs.h projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/sys/taskq.h projects/pnfs-planb-server/sys/cddl/dev/profile/profile.c projects/pnfs-planb-server/sys/compat/freebsd32/freebsd32_systrace_args.c projects/pnfs-planb-server/sys/compat/freebsd32/syscalls.master projects/pnfs-planb-server/sys/compat/linuxkpi/common/include/asm/atomic.h projects/pnfs-planb-server/sys/compat/linuxkpi/common/include/asm/msr.h projects/pnfs-planb-server/sys/compat/linuxkpi/common/include/linux/device.h projects/pnfs-planb-server/sys/compat/linuxkpi/common/include/linux/ktime.h projects/pnfs-planb-server/sys/compat/linuxkpi/common/include/linux/list.h projects/pnfs-planb-server/sys/compat/linuxkpi/common/include/linux/math64.h projects/pnfs-planb-server/sys/compat/linuxkpi/common/include/linux/mm.h projects/pnfs-planb-server/sys/compat/linuxkpi/common/include/linux/module.h projects/pnfs-planb-server/sys/compat/linuxkpi/common/include/linux/sched.h projects/pnfs-planb-server/sys/compat/linuxkpi/common/include/linux/slab.h projects/pnfs-planb-server/sys/compat/linuxkpi/common/include/linux/time.h projects/pnfs-planb-server/sys/compat/linuxkpi/common/include/linux/timer.h projects/pnfs-planb-server/sys/compat/linuxkpi/common/include/linux/types.h projects/pnfs-planb-server/sys/compat/linuxkpi/common/include/linux/wait.h projects/pnfs-planb-server/sys/compat/linuxkpi/common/include/linux/workqueue.h projects/pnfs-planb-server/sys/compat/linuxkpi/common/src/linux_schedule.c projects/pnfs-planb-server/sys/compat/linuxkpi/common/src/linux_work.c projects/pnfs-planb-server/sys/conf/NOTES projects/pnfs-planb-server/sys/conf/kern.mk projects/pnfs-planb-server/sys/conf/options projects/pnfs-planb-server/sys/contrib/dev/acpica/changes.txt projects/pnfs-planb-server/sys/contrib/dev/acpica/common/acfileio.c projects/pnfs-planb-server/sys/contrib/dev/acpica/common/dmtbdump2.c projects/pnfs-planb-server/sys/contrib/dev/acpica/common/dmtbinfo2.c projects/pnfs-planb-server/sys/contrib/dev/acpica/compiler/aslload.c projects/pnfs-planb-server/sys/contrib/dev/acpica/compiler/dttable1.c projects/pnfs-planb-server/sys/contrib/dev/acpica/compiler/dttemplate.h projects/pnfs-planb-server/sys/contrib/dev/acpica/components/debugger/dbnames.c projects/pnfs-planb-server/sys/contrib/dev/acpica/components/debugger/dbobject.c projects/pnfs-planb-server/sys/contrib/dev/acpica/components/dispatcher/dsdebug.c projects/pnfs-planb-server/sys/contrib/dev/acpica/components/executer/exconfig.c projects/pnfs-planb-server/sys/contrib/dev/acpica/components/namespace/nsdump.c projects/pnfs-planb-server/sys/contrib/dev/acpica/components/parser/psloop.c projects/pnfs-planb-server/sys/contrib/dev/acpica/components/parser/psobject.c projects/pnfs-planb-server/sys/contrib/dev/acpica/components/parser/pswalk.c projects/pnfs-planb-server/sys/contrib/dev/acpica/components/utilities/uterror.c projects/pnfs-planb-server/sys/contrib/dev/acpica/include/acdisasm.h projects/pnfs-planb-server/sys/contrib/dev/acpica/include/aclocal.h projects/pnfs-planb-server/sys/contrib/dev/acpica/include/acoutput.h projects/pnfs-planb-server/sys/contrib/dev/acpica/include/acpixf.h projects/pnfs-planb-server/sys/contrib/dev/acpica/include/actbinfo.h projects/pnfs-planb-server/sys/contrib/dev/acpica/include/actbl2.h projects/pnfs-planb-server/sys/contrib/dev/ath/ath_hal/ar9300/ar9300_ani.c projects/pnfs-planb-server/sys/contrib/dev/ath/ath_hal/ar9300/ar9300_freebsd.c projects/pnfs-planb-server/sys/contrib/dev/ath/ath_hal/ar9300/ar9300_reset.c projects/pnfs-planb-server/sys/contrib/zstd/lib/freebsd/stdlib.h projects/pnfs-planb-server/sys/dev/acpica/acpi.c projects/pnfs-planb-server/sys/dev/ath/ath_hal/ar5212/ar5212_ani.c projects/pnfs-planb-server/sys/dev/ath/ath_hal/ar5416/ar5416_ani.c projects/pnfs-planb-server/sys/dev/ath/ath_hal/ar5416/ar5416_cal.c projects/pnfs-planb-server/sys/dev/evdev/input-event-codes.h projects/pnfs-planb-server/sys/dev/evdev/input.h projects/pnfs-planb-server/sys/dev/hwpmc/hwpmc_amd.c projects/pnfs-planb-server/sys/dev/hwpmc/hwpmc_arm.c projects/pnfs-planb-server/sys/dev/hwpmc/hwpmc_arm64.c projects/pnfs-planb-server/sys/dev/hwpmc/hwpmc_armv7.c projects/pnfs-planb-server/sys/dev/hwpmc/hwpmc_core.c projects/pnfs-planb-server/sys/dev/hwpmc/hwpmc_core.h projects/pnfs-planb-server/sys/dev/hwpmc/hwpmc_e500.c projects/pnfs-planb-server/sys/dev/hwpmc/hwpmc_logging.c projects/pnfs-planb-server/sys/dev/hwpmc/hwpmc_mips.c projects/pnfs-planb-server/sys/dev/hwpmc/hwpmc_mod.c projects/pnfs-planb-server/sys/dev/hwpmc/hwpmc_mpc7xxx.c projects/pnfs-planb-server/sys/dev/hwpmc/hwpmc_ppc970.c projects/pnfs-planb-server/sys/dev/hwpmc/hwpmc_soft.c projects/pnfs-planb-server/sys/dev/liquidio/lio_bsd.h projects/pnfs-planb-server/sys/dev/md/md.c projects/pnfs-planb-server/sys/dev/ocs_fc/ocs_mgmt.c projects/pnfs-planb-server/sys/dev/ofw/ofw_fdt.c projects/pnfs-planb-server/sys/dev/puc/pucdata.c projects/pnfs-planb-server/sys/dev/usb/serial/u3g.c projects/pnfs-planb-server/sys/dev/usb/usbdevs projects/pnfs-planb-server/sys/dev/vnic/thunder_bgx_fdt.c projects/pnfs-planb-server/sys/i386/i386/trap.c projects/pnfs-planb-server/sys/i386/include/cpufunc.h projects/pnfs-planb-server/sys/i386/include/pmc_mdep.h projects/pnfs-planb-server/sys/kern/kern_exec.c projects/pnfs-planb-server/sys/kern/kern_kthread.c projects/pnfs-planb-server/sys/kern/kern_malloc.c projects/pnfs-planb-server/sys/kern/kern_mutex.c projects/pnfs-planb-server/sys/kern/kern_pmc.c projects/pnfs-planb-server/sys/kern/kern_rmlock.c projects/pnfs-planb-server/sys/kern/kern_rwlock.c projects/pnfs-planb-server/sys/kern/kern_thr.c projects/pnfs-planb-server/sys/kern/kern_thread.c projects/pnfs-planb-server/sys/kern/subr_counter.c projects/pnfs-planb-server/sys/kern/subr_pidctrl.c projects/pnfs-planb-server/sys/kern/subr_trap.c projects/pnfs-planb-server/sys/kern/subr_turnstile.c projects/pnfs-planb-server/sys/kern/syscalls.master projects/pnfs-planb-server/sys/kern/systrace_args.c projects/pnfs-planb-server/sys/kern/uipc_debug.c projects/pnfs-planb-server/sys/kern/uipc_sockbuf.c projects/pnfs-planb-server/sys/kern/uipc_socket.c projects/pnfs-planb-server/sys/kern/uipc_usrreq.c projects/pnfs-planb-server/sys/libkern/strchr.c projects/pnfs-planb-server/sys/libkern/strrchr.c projects/pnfs-planb-server/sys/mips/atheros/apb.c projects/pnfs-planb-server/sys/mips/cavium/octeon_pmc.c projects/pnfs-planb-server/sys/modules/Makefile projects/pnfs-planb-server/sys/modules/tcp/Makefile projects/pnfs-planb-server/sys/net/if_gif.c projects/pnfs-planb-server/sys/net/if_gif.h projects/pnfs-planb-server/sys/net/if_gre.c projects/pnfs-planb-server/sys/net/if_gre.h projects/pnfs-planb-server/sys/net/if_loop.c projects/pnfs-planb-server/sys/net/if_me.c projects/pnfs-planb-server/sys/net/if_stf.c projects/pnfs-planb-server/sys/net/iflib.c projects/pnfs-planb-server/sys/netinet/in_gif.c projects/pnfs-planb-server/sys/netinet/in_pcb.c projects/pnfs-planb-server/sys/netinet/in_pcb.h projects/pnfs-planb-server/sys/netinet/ip_encap.c projects/pnfs-planb-server/sys/netinet/ip_encap.h projects/pnfs-planb-server/sys/netinet/ip_gre.c projects/pnfs-planb-server/sys/netinet/ip_mroute.c projects/pnfs-planb-server/sys/netinet/ip_output.c projects/pnfs-planb-server/sys/netinet/libalias/alias_mod.h projects/pnfs-planb-server/sys/netinet/pim_var.h projects/pnfs-planb-server/sys/netinet/sctp_input.c projects/pnfs-planb-server/sys/netinet/sctp_usrreq.c projects/pnfs-planb-server/sys/netinet/tcp.h projects/pnfs-planb-server/sys/netinet/tcp_fsm.h projects/pnfs-planb-server/sys/netinet/tcp_hpts.c projects/pnfs-planb-server/sys/netinet/tcp_log_buf.h projects/pnfs-planb-server/sys/netinet/tcp_output.c projects/pnfs-planb-server/sys/netinet/tcp_stacks/fastpath.c projects/pnfs-planb-server/sys/netinet/tcp_timer.c projects/pnfs-planb-server/sys/netinet/tcp_timer.h projects/pnfs-planb-server/sys/netinet/tcp_var.h projects/pnfs-planb-server/sys/netinet/udp_usrreq.c projects/pnfs-planb-server/sys/netinet6/in6_gif.c projects/pnfs-planb-server/sys/netinet6/in6_pcb.c projects/pnfs-planb-server/sys/netinet6/in6_src.c projects/pnfs-planb-server/sys/netinet6/ip6_gre.c projects/pnfs-planb-server/sys/netinet6/ip6_mroute.c projects/pnfs-planb-server/sys/netinet6/ip6_output.c projects/pnfs-planb-server/sys/netinet6/pim6_var.h projects/pnfs-planb-server/sys/netinet6/scope6.c projects/pnfs-planb-server/sys/netinet6/scope6_var.h projects/pnfs-planb-server/sys/netinet6/udp6_usrreq.c projects/pnfs-planb-server/sys/netipsec/key.c projects/pnfs-planb-server/sys/netipsec/xform_ah.c projects/pnfs-planb-server/sys/netipsec/xform_ipcomp.c projects/pnfs-planb-server/sys/netpfil/ipfw/ip_fw_sockopt.c projects/pnfs-planb-server/sys/netpfil/pf/pf.c projects/pnfs-planb-server/sys/powerpc/ofw/ofw_machdep.c projects/pnfs-planb-server/sys/powerpc/powerpc/interrupt.c projects/pnfs-planb-server/sys/sys/kern_prefetch.h projects/pnfs-planb-server/sys/sys/malloc.h projects/pnfs-planb-server/sys/sys/mbuf.h projects/pnfs-planb-server/sys/sys/param.h projects/pnfs-planb-server/sys/sys/pmc.h projects/pnfs-planb-server/sys/sys/pmckern.h projects/pnfs-planb-server/sys/sys/pmclog.h projects/pnfs-planb-server/sys/sys/proc.h projects/pnfs-planb-server/sys/sys/queue.h projects/pnfs-planb-server/sys/sys/sockbuf.h projects/pnfs-planb-server/sys/sys/socket.h projects/pnfs-planb-server/sys/sys/socketvar.h projects/pnfs-planb-server/sys/sys/time.h projects/pnfs-planb-server/sys/sys/turnstile.h projects/pnfs-planb-server/sys/vm/uma.h projects/pnfs-planb-server/sys/vm/uma_core.c projects/pnfs-planb-server/sys/vm/uma_int.h projects/pnfs-planb-server/sys/vm/vm_map.c projects/pnfs-planb-server/sys/vm/vm_mmap.c projects/pnfs-planb-server/sys/vm/vm_page.c projects/pnfs-planb-server/sys/vm/vm_pageout.c projects/pnfs-planb-server/sys/vm/vm_unix.c projects/pnfs-planb-server/sys/x86/x86/cpu_machdep.c projects/pnfs-planb-server/sys/x86/x86/mp_x86.c Directory Properties: projects/pnfs-planb-server/sys/ (props changed) projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/ (props changed) projects/pnfs-planb-server/sys/contrib/dev/acpica/ (props changed) projects/pnfs-planb-server/sys/contrib/zstd/ (props changed) Modified: projects/pnfs-planb-server/sys/amd64/amd64/machdep.c ============================================================================== --- projects/pnfs-planb-server/sys/amd64/amd64/machdep.c Sat Jun 9 23:38:22 2018 (r334904) +++ projects/pnfs-planb-server/sys/amd64/amd64/machdep.c Sat Jun 9 23:45:05 2018 (r334905) @@ -1248,15 +1248,6 @@ getmemsize(caddr_t kmdp, u_int64_t first) } /* - * Make hole for "AP -> long mode" bootstrap code. The - * mp_bootaddress vector is only available when the kernel - * is configured to support APs and APs for the system start - * in real mode mode (e.g. SMP bare metal). - */ - if (init_ops.mp_bootaddress) - init_ops.mp_bootaddress(physmap, &physmap_idx); - - /* * Maxmem isn't the "maximum memory", it's one larger than the * highest page of the physical address space. It should be * called something like "Maxphyspage". We may adjust this @@ -1293,6 +1284,15 @@ getmemsize(caddr_t kmdp, u_int64_t first) if (atop(physmap[physmap_idx + 1]) != Maxmem && (boothowto & RB_VERBOSE)) printf("Physical memory use set to %ldK\n", Maxmem * 4); + + /* + * Make hole for "AP -> long mode" bootstrap code. The + * mp_bootaddress vector is only available when the kernel + * is configured to support APs and APs for the system start + * in real mode mode (e.g. SMP bare metal). + */ + if (init_ops.mp_bootaddress) + init_ops.mp_bootaddress(physmap, &physmap_idx); /* call pmap initialization to make new kernel address space */ pmap_bootstrap(&first); Modified: projects/pnfs-planb-server/sys/amd64/amd64/mp_machdep.c ============================================================================== --- projects/pnfs-planb-server/sys/amd64/amd64/mp_machdep.c Sat Jun 9 23:38:22 2018 (r334904) +++ projects/pnfs-planb-server/sys/amd64/amd64/mp_machdep.c Sat Jun 9 23:45:05 2018 (r334905) @@ -113,12 +113,16 @@ mp_bootaddress(vm_paddr_t *physmap, unsigned int *phys allocated = false; for (i = *physmap_idx; i <= *physmap_idx; i -= 2) { /* - * Find a memory region big enough below the 4GB boundary to - * store the initial page tables. Note that it needs to be - * aligned to a page boundary. + * Find a memory region big enough below the 4GB + * boundary to store the initial page tables. Region + * must be mapped by the direct map. + * + * Note that it needs to be aligned to a page + * boundary. */ - if (physmap[i] >= GiB(4) || - (physmap[i + 1] - round_page(physmap[i])) < (PAGE_SIZE * 3)) + if (physmap[i] >= GiB(4) || physmap[i + 1] - + round_page(physmap[i]) < PAGE_SIZE * 3 || + physmap[i + 1] > Maxmem) continue; allocated = true; Modified: projects/pnfs-planb-server/sys/amd64/amd64/pmap.c ============================================================================== --- projects/pnfs-planb-server/sys/amd64/amd64/pmap.c Sat Jun 9 23:38:22 2018 (r334904) +++ projects/pnfs-planb-server/sys/amd64/amd64/pmap.c Sat Jun 9 23:45:05 2018 (r334905) @@ -513,7 +513,7 @@ pmap_delayed_invl_finished(void) pmap_invl_gen = invl_gen->gen; if (ts != NULL) { turnstile_broadcast(ts, TS_SHARED_QUEUE); - turnstile_unpend(ts, TS_SHARED_LOCK); + turnstile_unpend(ts); } turnstile_chain_unlock(&invl_gen_ts); } else { Modified: projects/pnfs-planb-server/sys/amd64/amd64/support.S ============================================================================== --- projects/pnfs-planb-server/sys/amd64/amd64/support.S Sat Jun 9 23:38:22 2018 (r334904) +++ projects/pnfs-planb-server/sys/amd64/amd64/support.S Sat Jun 9 23:45:05 2018 (r334905) @@ -40,37 +40,31 @@ .text -/* - * bcopy family - * void bzero(void *buf, u_int len) - */ - -/* done */ -ENTRY(bzero) +/* Address: %rdi */ +ENTRY(pagezero) PUSH_FRAME_POINTER - movq %rsi,%rcx + movq $PAGE_SIZE/8,%rcx xorl %eax,%eax - shrq $3,%rcx rep stosq - movq %rsi,%rcx - andq $7,%rcx - rep - stosb POP_FRAME_POINTER ret -END(bzero) +END(pagezero) -/* Address: %rdi */ -ENTRY(pagezero) +/* + * pagecopy(%rdi=from, %rsi=to) + */ +ENTRY(pagecopy) PUSH_FRAME_POINTER movq $PAGE_SIZE/8,%rcx - xorl %eax,%eax + movq %rdi,%r9 + movq %rsi,%rdi + movq %r9,%rsi rep - stosq + movsq POP_FRAME_POINTER ret -END(pagezero) +END(pagecopy) /* Address: %rdi */ ENTRY(sse2_pagezero) @@ -96,95 +90,7 @@ ENTRY(sse2_pagezero) ret END(sse2_pagezero) -ENTRY(bcmp) - PUSH_FRAME_POINTER - test %rdx,%rdx - je 1f - cmpq $64,%rdx - jg 4f - - xor %ecx,%ecx -2: - movzbl (%rdi,%rcx,1),%eax - movzbl (%rsi,%rcx,1),%r8d - cmp %r8b,%al - jne 3f - add $0x1,%rcx - cmp %rcx,%rdx - jne 2b -1: - xor %eax,%eax - POP_FRAME_POINTER - retq -3: - mov $1,%eax - POP_FRAME_POINTER - retq -4: - movq %rdx,%rcx - shrq $3,%rcx - repe - cmpsq - jne 5f - - movq %rdx,%rcx - andq $7,%rcx - repe - cmpsb -5: - setne %al - movsbl %al,%eax - POP_FRAME_POINTER - ret -END(bcmp) - /* - * bcopy(src, dst, cnt) - * rdi, rsi, rdx - * ws@tools.de (Wolfgang Solfrank, TooLs GmbH) +49-228-985800 - */ -ENTRY(bcopy) - PUSH_FRAME_POINTER - xchgq %rsi,%rdi - movq %rdx,%rcx - - movq %rdi,%rax - subq %rsi,%rax - cmpq %rcx,%rax /* overlapping && src < dst? */ - jb 1f - - shrq $3,%rcx /* copy by 64-bit words */ - rep - movsq - movq %rdx,%rcx - andq $7,%rcx /* any bytes left? */ - rep - movsb - POP_FRAME_POINTER - ret - - /* ALIGN_TEXT */ -1: - addq %rcx,%rdi /* copy backwards */ - addq %rcx,%rsi - decq %rdi - decq %rsi - andq $7,%rcx /* any fractional bytes? */ - std - rep - movsb - movq %rdx,%rcx /* copy remainder by 32-bit words */ - shrq $3,%rcx - subq $7,%rsi - subq $7,%rdi - rep - movsq - cld - POP_FRAME_POINTER - ret -END(bcopy) - -/* * memmove(dst, src, cnt) * rdi, rsi, rdx * Adapted from bcopy written by: @@ -271,8 +177,9 @@ ENTRY(memset) PUSH_FRAME_POINTER movq %rdi,%r9 movq %rdx,%rcx + movzbq %sil,%r8 movabs $0x0101010101010101,%rax - imulq %rsi,%rax + imulq %r8,%rax shrq $3,%rcx rep stosq @@ -289,21 +196,6 @@ ENTRY(memset) POP_FRAME_POINTER ret END(memset) - -/* - * pagecopy(%rdi=from, %rsi=to) - */ -ENTRY(pagecopy) - PUSH_FRAME_POINTER - movq $PAGE_SIZE/8,%rcx - movq %rdi,%r9 - movq %rsi,%rdi - movq %r9,%rsi - rep - movsq - POP_FRAME_POINTER - ret -END(pagecopy) /* fillw(pat, base, cnt) */ /* %rdi,%rsi, %rdx */ Modified: projects/pnfs-planb-server/sys/amd64/amd64/trap.c ============================================================================== --- projects/pnfs-planb-server/sys/amd64/amd64/trap.c Sat Jun 9 23:38:22 2018 (r334904) +++ projects/pnfs-planb-server/sys/amd64/amd64/trap.c Sat Jun 9 23:45:05 2018 (r334905) @@ -214,7 +214,7 @@ trap(struct trapframe *frame) * the NMI was consumed by it and we can return immediately. */ if (pmc_intr != NULL && - (*pmc_intr)(PCPU_GET(cpuid), frame) != 0) + (*pmc_intr)(frame) != 0) return; #endif @@ -770,7 +770,8 @@ trap_pfault(struct trapframe *frame, int usermode) * If nx protection of the usermode portion of kernel page * tables caused trap, panic. */ - if (pti && usermode && pg_nx != 0 && (frame->tf_err & (PGEX_P | PGEX_W | + if (PCPU_GET(curpmap)->pm_ucr3 != PMAP_NO_CR3 && usermode && + pg_nx != 0 && (frame->tf_err & (PGEX_P | PGEX_W | PGEX_U | PGEX_I)) == (PGEX_P | PGEX_U | PGEX_I) && (curpcb->pcb_saved_ucr3 & ~CR3_PCID_MASK)== (PCPU_GET(curpmap)->pm_cr3 & ~CR3_PCID_MASK)) Modified: projects/pnfs-planb-server/sys/amd64/include/cpufunc.h ============================================================================== --- projects/pnfs-planb-server/sys/amd64/include/cpufunc.h Sat Jun 9 23:38:22 2018 (r334904) +++ projects/pnfs-planb-server/sys/amd64/include/cpufunc.h Sat Jun 9 23:45:05 2018 (r334905) @@ -386,6 +386,15 @@ rdtsc(void) return (low | ((uint64_t)high << 32)); } +static __inline uint64_t +rdtscp(void) +{ + uint32_t low, high; + + __asm __volatile("rdtscp" : "=a" (low), "=d" (high) : : "ecx"); + return (low | ((uint64_t)high << 32)); +} + static __inline uint32_t rdtsc32(void) { Modified: projects/pnfs-planb-server/sys/amd64/include/pmc_mdep.h ============================================================================== --- projects/pnfs-planb-server/sys/amd64/include/pmc_mdep.h Sat Jun 9 23:38:22 2018 (r334904) +++ projects/pnfs-planb-server/sys/amd64/include/pmc_mdep.h Sat Jun 9 23:45:05 2018 (r334905) @@ -73,7 +73,6 @@ struct pmc_mdep; union pmc_md_op_pmcallocate { struct pmc_md_amd_op_pmcallocate pm_amd; - struct pmc_md_iaf_op_pmcallocate pm_iaf; struct pmc_md_iap_op_pmcallocate pm_iap; struct pmc_md_ucf_op_pmcallocate pm_ucf; struct pmc_md_ucp_op_pmcallocate pm_ucp; Modified: projects/pnfs-planb-server/sys/arm/arm/pmu.c ============================================================================== --- projects/pnfs-planb-server/sys/arm/arm/pmu.c Sat Jun 9 23:38:22 2018 (r334904) +++ projects/pnfs-planb-server/sys/arm/arm/pmu.c Sat Jun 9 23:45:05 2018 (r334905) @@ -123,7 +123,7 @@ pmu_intr(void *arg) /* Only call into the HWPMC framework if we know there is work. */ if (r != 0 && pmc_intr) { tf = arg; - (*pmc_intr)(PCPU_GET(cpuid), tf); + (*pmc_intr)(tf); } #endif Modified: projects/pnfs-planb-server/sys/arm/conf/GENERIC-NODEBUG ============================================================================== --- projects/pnfs-planb-server/sys/arm/conf/GENERIC-NODEBUG Sat Jun 9 23:38:22 2018 (r334904) +++ projects/pnfs-planb-server/sys/arm/conf/GENERIC-NODEBUG Sat Jun 9 23:45:05 2018 (r334905) @@ -34,7 +34,5 @@ nooptions INVARIANTS nooptions INVARIANT_SUPPORT nooptions WITNESS nooptions WITNESS_SKIPSPIN -nooptions BUF_TRACKING nooptions DEADLKRES -nooptions FULL_BUF_TRACKING Modified: projects/pnfs-planb-server/sys/arm/xscale/ixp425/files.ixp425 ============================================================================== --- projects/pnfs-planb-server/sys/arm/xscale/ixp425/files.ixp425 Sat Jun 9 23:38:22 2018 (r334904) +++ projects/pnfs-planb-server/sys/arm/xscale/ixp425/files.ixp425 Sat Jun 9 23:45:05 2018 (r334905) @@ -13,7 +13,6 @@ arm/xscale/ixp425/uart_bus_ixp425.c optional uart arm/xscale/ixp425/ixp425_a4x_space.c optional uart arm/xscale/ixp425/ixp425_a4x_io.S optional uart dev/cfi/cfi_bus_ixp4xx.c optional cfi -dev/hwpmc/hwpmc_xscale.c optional hwpmc dev/uart/uart_dev_ns8250.c optional uart # # NPE-based Ethernet support (requires qmgr also). Modified: projects/pnfs-planb-server/sys/arm64/arm64/swtch.S ============================================================================== --- projects/pnfs-planb-server/sys/arm64/arm64/swtch.S Sat Jun 9 23:38:22 2018 (r334904) +++ projects/pnfs-planb-server/sys/arm64/arm64/swtch.S Sat Jun 9 23:45:05 2018 (r334905) @@ -165,10 +165,9 @@ ENTRY(cpu_switch) mov x0, x19 /* - * Release the old thread. This doesn't need to be a store-release - * as the above dsb instruction will provide release semantics. + * Release the old thread. */ - str x2, [x0, #TD_LOCK] + stlr x2, [x0, #TD_LOCK] #if defined(SCHED_ULE) && defined(SMP) /* Spin if TD_LOCK points to a blocked_lock */ ldr x2, =_C_LABEL(blocked_lock) Modified: projects/pnfs-planb-server/sys/arm64/conf/GENERIC-NODEBUG ============================================================================== --- projects/pnfs-planb-server/sys/arm64/conf/GENERIC-NODEBUG Sat Jun 9 23:38:22 2018 (r334904) +++ projects/pnfs-planb-server/sys/arm64/conf/GENERIC-NODEBUG Sat Jun 9 23:45:05 2018 (r334905) @@ -34,7 +34,5 @@ nooptions INVARIANTS nooptions INVARIANT_SUPPORT nooptions WITNESS nooptions WITNESS_SKIPSPIN -nooptions BUF_TRACKING nooptions DEADLKRES -nooptions FULL_BUF_TRACKING nooptions USB_DEBUG Modified: projects/pnfs-planb-server/sys/cam/mmc/mmc_da.c ============================================================================== --- projects/pnfs-planb-server/sys/cam/mmc/mmc_da.c Sat Jun 9 23:38:22 2018 (r334904) +++ projects/pnfs-planb-server/sys/cam/mmc/mmc_da.c Sat Jun 9 23:45:05 2018 (r334905) @@ -171,7 +171,7 @@ static void sdda_start_init_task(void *context, int pe static void sdda_process_mmc_partitions(struct cam_periph *periph, union ccb *start_ccb); static uint32_t sdda_get_host_caps(struct cam_periph *periph, union ccb *ccb); static void sdda_init_switch_part(struct cam_periph *periph, union ccb *start_ccb, u_int part); - +static int mmc_select_card(struct cam_periph *periph, union ccb *ccb, uint32_t rca); static inline uint32_t mmc_get_sector_size(struct cam_periph *periph) {return MMC_SECTOR_SIZE;} /* TODO: actually issue GET_TRAN_SETTINGS to get R/O status */ @@ -901,6 +901,38 @@ mmc_switch_fill_mmcio(union ccb *ccb, } static int +mmc_select_card(struct cam_periph *periph, union ccb *ccb, uint32_t rca) +{ + int flags; + + flags = (rca ? MMC_RSP_R1B : MMC_RSP_NONE) | MMC_CMD_AC; + cam_fill_mmcio(&ccb->mmcio, + /*retries*/ 0, + /*cbfcnp*/ NULL, + /*flags*/ CAM_DIR_IN, + /*mmc_opcode*/ MMC_SELECT_CARD, + /*mmc_arg*/ rca << 16, + /*mmc_flags*/ flags, + /*mmc_data*/ NULL, + /*timeout*/ 0); + + cam_periph_runccb(ccb, sddaerror, CAM_FLAG_NONE, /*sense_flags*/0, NULL); + + if (((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP)) { + if (ccb->mmcio.cmd.error != 0) { + CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_PERIPH, + ("%s: MMC_SELECT command failed", __func__)); + return EIO; + } + return 0; /* Normal return */ + } else { + CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_PERIPH, + ("%s: CAM request failed\n", __func__)); + return EIO; + } +} + +static int mmc_switch(struct cam_periph *periph, union ccb *ccb, uint8_t set, uint8_t index, uint8_t value, u_int timeout) { @@ -953,18 +985,24 @@ mmc_sd_switch(struct cam_periph *periph, union ccb *cc uint8_t *res) { struct mmc_data mmc_d; + uint32_t arg; memset(res, 0, 64); mmc_d.len = 64; mmc_d.data = res; mmc_d.flags = MMC_DATA_READ; + arg = mode << 31; /* 0 - check, 1 - set */ + arg |= 0x00FFFFFF; + arg &= ~(0xF << (grp * 4)); + arg |= value << (grp * 4); + cam_fill_mmcio(&ccb->mmcio, /*retries*/ 0, /*cbfcnp*/ NULL, /*flags*/ CAM_DIR_IN, /*mmc_opcode*/ SD_SWITCH_FUNC, - /*mmc_arg*/ mode << 31, + /*mmc_arg*/ arg, /*mmc_flags*/ MMC_RSP_R1 | MMC_CMD_ADTC, /*mmc_data*/ &mmc_d, /*timeout*/ 0); @@ -1273,6 +1311,19 @@ sdda_start_init(void *context, union ccb *start_ccb) CAM_DEBUG(periph->path, CAM_DEBUG_PERIPH, ("Card supports HS\n")); softc->card_f_max = SD_HS_MAX; } + + /* + * We deselect then reselect the card here. Some cards + * become unselected and timeout with the above two + * commands, although the state tables / diagrams in the + * standard suggest they go back to the transfer state. + * Other cards don't become deselected, and if we + * attempt to blindly re-select them, we get timeout + * errors from some controllers. So we deselect then + * reselect to handle all situations. + */ + mmc_select_card(periph, start_ccb, 0); + mmc_select_card(periph, start_ccb, get_rca(periph)); } else { CAM_DEBUG(periph->path, CAM_DEBUG_PERIPH, ("Not trying the switch\n")); goto finish_hs_tests; @@ -1293,6 +1344,15 @@ finish_hs_tests: f_max = min(host_f_max, softc->card_f_max); CAM_DEBUG(periph->path, CAM_DEBUG_PERIPH, ("Set SD freq to %d MHz (min out of host f=%d MHz and card f=%d MHz)\n", f_max / 1000000, host_f_max / 1000000, softc->card_f_max / 1000000)); + /* Enable high-speed timing on the card */ + if (f_max > 25000000) { + err = mmc_set_timing(periph, start_ccb, bus_timing_hs); + if (err != MMC_ERR_NONE) { + CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("Cannot switch card to high-speed mode")); + f_max = 25000000; + } + } + /* Set frequency on the controller */ start_ccb->ccb_h.func_code = XPT_SET_TRAN_SETTINGS; start_ccb->ccb_h.flags = CAM_DIR_NONE; start_ccb->ccb_h.retry_count = 0; @@ -1326,12 +1386,6 @@ finish_hs_tests: bus_width_str(max_host_bus_width), bus_width_str(max_card_bus_width))); sdda_set_bus_width(periph, start_ccb, desired_bus_width); - - if (f_max > 25000000) { - err = mmc_set_timing(periph, start_ccb, bus_timing_hs); - if (err != MMC_ERR_NONE) - CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("Cannot switch card to high-speed mode")); - } softc->state = SDDA_STATE_NORMAL; Modified: projects/pnfs-planb-server/sys/cddl/compat/opensolaris/kern/opensolaris_taskq.c ============================================================================== --- projects/pnfs-planb-server/sys/cddl/compat/opensolaris/kern/opensolaris_taskq.c Sat Jun 9 23:38:22 2018 (r334904) +++ projects/pnfs-planb-server/sys/cddl/compat/opensolaris/kern/opensolaris_taskq.c Sat Jun 9 23:45:05 2018 (r334905) @@ -173,3 +173,9 @@ taskq_wait(taskq_t *tq) { taskqueue_drain_all(tq->tq_queue); } + +void +taskq_wait_id(taskq_t *tq, taskqid_t id) +{ + taskq_wait(tq); +} Modified: projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c ============================================================================== --- projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c Sat Jun 9 23:38:22 2018 (r334904) +++ projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c Sat Jun 9 23:45:05 2018 (r334905) @@ -339,7 +339,8 @@ int arc_no_grow_shift = 5; * minimum lifespan of a prefetch block in clock ticks * (initialized in arc_init()) */ -static int arc_min_prefetch_lifespan; +static int zfs_arc_min_prefetch_ms = 1; +static int zfs_arc_min_prescient_prefetch_ms = 6; /* * If this percent of memory is free, don't throttle. @@ -779,8 +780,9 @@ typedef struct arc_stats { kstat_named_t arcstat_meta_limit; kstat_named_t arcstat_meta_max; kstat_named_t arcstat_meta_min; - kstat_named_t arcstat_sync_wait_for_async; + kstat_named_t arcstat_async_upgrade_sync; kstat_named_t arcstat_demand_hit_predictive_prefetch; + kstat_named_t arcstat_demand_hit_prescient_prefetch; } arc_stats_t; static arc_stats_t arc_stats = { @@ -877,8 +879,9 @@ static arc_stats_t arc_stats = { { "arc_meta_limit", KSTAT_DATA_UINT64 }, { "arc_meta_max", KSTAT_DATA_UINT64 }, { "arc_meta_min", KSTAT_DATA_UINT64 }, - { "sync_wait_for_async", KSTAT_DATA_UINT64 }, + { "async_upgrade_sync", KSTAT_DATA_UINT64 }, { "demand_hit_predictive_prefetch", KSTAT_DATA_UINT64 }, + { "demand_hit_prescient_prefetch", KSTAT_DATA_UINT64 }, }; #define ARCSTAT(stat) (arc_stats.stat.value.ui64) @@ -974,22 +977,23 @@ typedef struct arc_callback arc_callback_t; struct arc_callback { void *acb_private; - arc_done_func_t *acb_done; + arc_read_done_func_t *acb_done; arc_buf_t *acb_buf; boolean_t acb_compressed; zio_t *acb_zio_dummy; + zio_t *acb_zio_head; arc_callback_t *acb_next; }; typedef struct arc_write_callback arc_write_callback_t; struct arc_write_callback { - void *awcb_private; - arc_done_func_t *awcb_ready; - arc_done_func_t *awcb_children_ready; - arc_done_func_t *awcb_physdone; - arc_done_func_t *awcb_done; - arc_buf_t *awcb_buf; + void *awcb_private; + arc_write_done_func_t *awcb_ready; + arc_write_done_func_t *awcb_children_ready; + arc_write_done_func_t *awcb_physdone; + arc_write_done_func_t *awcb_done; + arc_buf_t *awcb_buf; }; /* @@ -1229,6 +1233,8 @@ sysctl_vfs_zfs_arc_min(SYSCTL_HANDLER_ARGS) #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) #define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_FLAG_IO_ERROR) #define HDR_PREFETCH(hdr) ((hdr)->b_flags & ARC_FLAG_PREFETCH) +#define HDR_PRESCIENT_PREFETCH(hdr) \ + ((hdr)->b_flags & ARC_FLAG_PRESCIENT_PREFETCH) #define HDR_COMPRESSION_ENABLED(hdr) \ ((hdr)->b_flags & ARC_FLAG_COMPRESSED_ARC) @@ -1392,6 +1398,11 @@ SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, mfu_ghost_data_esize, SYSCTL_UQUAD(_vfs_zfs, OID_AUTO, l2c_only_size, CTLFLAG_RD, &ARC_l2c_only.arcs_size.rc_count, 0, "size of mru state"); +SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_min_prfetch_ms, CTLFLAG_RW, + &zfs_arc_min_prefetch_ms, 0, "Min life of prefetch block in ms"); +SYSCTL_UINT(_vfs_zfs, OID_AUTO, arc_min_prescient_prefetch_ms, CTLFLAG_RW, + &zfs_arc_min_prescient_prefetch_ms, 0, "Min life oof prescient prefetched block in ms"); + /* * L2ARC Internals */ @@ -3544,6 +3555,8 @@ arc_evict_hdr(arc_buf_hdr_t *hdr, kmutex_t *hash_lock) { arc_state_t *evicted_state, *state; int64_t bytes_evicted = 0; + int min_lifetime = HDR_PRESCIENT_PREFETCH(hdr) ? + zfs_arc_min_prescient_prefetch_ms : zfs_arc_min_prefetch_ms; ASSERT(MUTEX_HELD(hash_lock)); ASSERT(HDR_HAS_L1HDR(hdr)); @@ -3596,8 +3609,7 @@ arc_evict_hdr(arc_buf_hdr_t *hdr, kmutex_t *hash_lock) /* prefetch buffers have a minimum lifespan */ if (HDR_IO_IN_PROGRESS(hdr) || ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) && - ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access < - arc_min_prefetch_lifespan)) { + ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access < min_lifetime * hz)) { ARCSTAT_BUMP(arcstat_evict_skip); return (bytes_evicted); } @@ -4968,13 +4980,15 @@ arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock) * - move the buffer to the head of the list if this is * another prefetch (to make it less likely to be evicted). */ - if (HDR_PREFETCH(hdr)) { + if (HDR_PREFETCH(hdr) || HDR_PRESCIENT_PREFETCH(hdr)) { if (refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) { /* link protected by hash lock */ ASSERT(multilist_link_active( &hdr->b_l1hdr.b_arc_node)); } else { - arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH); + arc_hdr_clear_flags(hdr, + ARC_FLAG_PREFETCH | + ARC_FLAG_PRESCIENT_PREFETCH); ARCSTAT_BUMP(arcstat_mru_hits); } hdr->b_l1hdr.b_arc_access = now; @@ -5005,10 +5019,13 @@ arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock) * MFU state. */ - if (HDR_PREFETCH(hdr)) { + if (HDR_PREFETCH(hdr) || HDR_PRESCIENT_PREFETCH(hdr)) { new_state = arc_mru; - if (refcount_count(&hdr->b_l1hdr.b_refcnt) > 0) - arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH); + if (refcount_count(&hdr->b_l1hdr.b_refcnt) > 0) { + arc_hdr_clear_flags(hdr, + ARC_FLAG_PREFETCH | + ARC_FLAG_PRESCIENT_PREFETCH); + } DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr); } else { new_state = arc_mfu; @@ -5029,11 +5046,7 @@ arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock) * If it was a prefetch, we will explicitly move it to * the head of the list now. */ - if ((HDR_PREFETCH(hdr)) != 0) { - ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); - /* link protected by hash_lock */ - ASSERT(multilist_link_active(&hdr->b_l1hdr.b_arc_node)); - } + ARCSTAT_BUMP(arcstat_mfu_hits); hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); } else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) { @@ -5044,12 +5057,11 @@ arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock) * MFU state. */ - if (HDR_PREFETCH(hdr)) { + if (HDR_PREFETCH(hdr) || HDR_PRESCIENT_PREFETCH(hdr)) { /* * This is a prefetch access... * move this block back to the MRU state. */ - ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt)); new_state = arc_mru; } @@ -5116,23 +5128,28 @@ arc_buf_access(arc_buf_t *buf) demand, prefetch, !HDR_ISTYPE_METADATA(hdr), data, metadata, hits); } -/* a generic arc_done_func_t which you can use */ +/* a generic arc_read_done_func_t which you can use */ /* ARGSUSED */ void -arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg) +arc_bcopy_func(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp, + arc_buf_t *buf, void *arg) { - if (zio == NULL || zio->io_error == 0) - bcopy(buf->b_data, arg, arc_buf_size(buf)); + if (buf == NULL) + return; + + bcopy(buf->b_data, arg, arc_buf_size(buf)); arc_buf_destroy(buf, arg); } -/* a generic arc_done_func_t */ +/* a generic arc_read_done_func_t */ +/* ARGSUSED */ void -arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg) +arc_getbuf_func(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp, + arc_buf_t *buf, void *arg) { arc_buf_t **bufp = arg; - if (zio && zio->io_error) { - arc_buf_destroy(buf, arg); + + if (buf == NULL) { *bufp = NULL; } else { *bufp = buf; @@ -5164,7 +5181,6 @@ arc_read_done(zio_t *zio) arc_callback_t *callback_list; arc_callback_t *acb; boolean_t freeable = B_FALSE; - boolean_t no_zio_error = (zio->io_error == 0); /* * The hdr was inserted into hash-table and removed from lists @@ -5190,7 +5206,7 @@ arc_read_done(zio_t *zio) ASSERT3P(hash_lock, !=, NULL); } - if (no_zio_error) { + if (zio->io_error == 0) { /* byteswap if necessary */ if (BP_SHOULD_BYTESWAP(zio->io_bp)) { if (BP_GET_LEVEL(zio->io_bp) > 0) { @@ -5211,7 +5227,8 @@ arc_read_done(zio_t *zio) callback_list = hdr->b_l1hdr.b_acb; ASSERT3P(callback_list, !=, NULL); - if (hash_lock && no_zio_error && hdr->b_l1hdr.b_state == arc_anon) { + if (hash_lock && zio->io_error == 0 && + hdr->b_l1hdr.b_state == arc_anon) { /* * Only call arc_access on anonymous buffers. This is because * if we've issued an I/O for an evicted buffer, we've already @@ -5232,14 +5249,21 @@ arc_read_done(zio_t *zio) if (!acb->acb_done) continue; - /* This is a demand read since prefetches don't use callbacks */ callback_cnt++; + if (zio->io_error != 0) + continue; + int error = arc_buf_alloc_impl(hdr, acb->acb_private, - acb->acb_compressed, no_zio_error, &acb->acb_buf); - if (no_zio_error) { - zio->io_error = error; + acb->acb_compressed, + B_TRUE, &acb->acb_buf); + if (error != 0) { + arc_buf_destroy(acb->acb_buf, acb->acb_private); + acb->acb_buf = NULL; } + + if (zio->io_error == 0) + zio->io_error = error; } hdr->b_l1hdr.b_acb = NULL; arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); @@ -5252,7 +5276,7 @@ arc_read_done(zio_t *zio) ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt) || callback_list != NULL); - if (no_zio_error) { + if (zio->io_error == 0) { arc_hdr_verify(hdr, zio->io_bp); } else { arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR); @@ -5285,8 +5309,10 @@ arc_read_done(zio_t *zio) /* execute each callback and free its structure */ while ((acb = callback_list) != NULL) { - if (acb->acb_done) - acb->acb_done(zio, acb->acb_buf, acb->acb_private); + if (acb->acb_done) { + acb->acb_done(zio, &zio->io_bookmark, zio->io_bp, + acb->acb_buf, acb->acb_private); + } if (acb->acb_zio_dummy != NULL) { acb->acb_zio_dummy->io_error = zio->io_error; @@ -5320,7 +5346,7 @@ arc_read_done(zio_t *zio) * for readers of this block. */ int -arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_done_func_t *done, +arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_read_done_func_t *done, void *private, zio_priority_t priority, int zio_flags, arc_flags_t *arc_flags, const zbookmark_phys_t *zb) { @@ -5329,7 +5355,8 @@ arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, a zio_t *rzio; uint64_t guid = spa_load_guid(spa); boolean_t compressed_read = (zio_flags & ZIO_FLAG_RAW) != 0; - + int rc = 0; + ASSERT(!BP_IS_EMBEDDED(bp) || BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA); @@ -5347,32 +5374,20 @@ top: *arc_flags |= ARC_FLAG_CACHED; if (HDR_IO_IN_PROGRESS(hdr)) { + zio_t *head_zio = hdr->b_l1hdr.b_acb->acb_zio_head; + ASSERT3P(head_zio, !=, NULL); if ((hdr->b_flags & ARC_FLAG_PRIO_ASYNC_READ) && priority == ZIO_PRIORITY_SYNC_READ) { /* - * This sync read must wait for an - * in-progress async read (e.g. a predictive - * prefetch). Async reads are queued - * separately at the vdev_queue layer, so - * this is a form of priority inversion. - * Ideally, we would "inherit" the demand - * i/o's priority by moving the i/o from - * the async queue to the synchronous queue, - * but there is currently no mechanism to do - * so. Track this so that we can evaluate - * the magnitude of this potential performance - * problem. - * - * Note that if the prefetch i/o is already - * active (has been issued to the device), - * the prefetch improved performance, because - * we issued it sooner than we would have - * without the prefetch. + * This is a sync read that needs to wait for + * an in-flight async read. Request that the + * zio have its priority upgraded. */ - DTRACE_PROBE1(arc__sync__wait__for__async, + zio_change_priority(head_zio, priority); + DTRACE_PROBE1(arc__async__upgrade__sync, arc_buf_hdr_t *, hdr); - ARCSTAT_BUMP(arcstat_sync_wait_for_async); + ARCSTAT_BUMP(arcstat_async_upgrade_sync); } if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) { arc_hdr_clear_flags(hdr, @@ -5399,6 +5414,7 @@ top: spa, NULL, NULL, NULL, zio_flags); ASSERT3P(acb->acb_done, !=, NULL); + acb->acb_zio_head = head_zio; acb->acb_next = hdr->b_l1hdr.b_acb; hdr->b_l1hdr.b_acb = acb; mutex_exit(hash_lock); @@ -5426,17 +5442,32 @@ top: arc_hdr_clear_flags(hdr, ARC_FLAG_PREDICTIVE_PREFETCH); } - ASSERT(!BP_IS_EMBEDDED(bp) || !BP_IS_HOLE(bp)); + if (hdr->b_flags & ARC_FLAG_PRESCIENT_PREFETCH) { + ARCSTAT_BUMP( + arcstat_demand_hit_prescient_prefetch); + arc_hdr_clear_flags(hdr, + ARC_FLAG_PRESCIENT_PREFETCH); + } + + ASSERT(!BP_IS_EMBEDDED(bp) || !BP_IS_HOLE(bp)); /* Get a buf with the desired data in it. */ - VERIFY0(arc_buf_alloc_impl(hdr, private, - compressed_read, B_TRUE, &buf)); + rc = arc_buf_alloc_impl(hdr, private, + compressed_read, B_TRUE, &buf); + if (rc != 0) { + arc_buf_destroy(buf, private); + buf = NULL; + } + ASSERT((zio_flags & ZIO_FLAG_SPECULATIVE) || + rc == 0 || rc != ENOENT); } else if (*arc_flags & ARC_FLAG_PREFETCH && refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) { arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH); } DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); arc_access(hdr, hash_lock); + if (*arc_flags & ARC_FLAG_PRESCIENT_PREFETCH) + arc_hdr_set_flags(hdr, ARC_FLAG_PRESCIENT_PREFETCH); if (*arc_flags & ARC_FLAG_L2CACHE) arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE); mutex_exit(hash_lock); @@ -5446,7 +5477,7 @@ top: data, metadata, hits); if (done) - done(NULL, buf, private); + done(NULL, zb, bp, buf, private); } else { uint64_t lsize = BP_GET_LSIZE(bp); uint64_t psize = BP_GET_PSIZE(bp); @@ -5520,6 +5551,9 @@ top: if (*arc_flags & ARC_FLAG_PREFETCH) arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH); + if (*arc_flags & ARC_FLAG_PRESCIENT_PREFETCH) + arc_hdr_set_flags(hdr, ARC_FLAG_PRESCIENT_PREFETCH); + if (*arc_flags & ARC_FLAG_L2CACHE) arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE); if (BP_GET_LEVEL(bp) > 0) @@ -5549,14 +5583,17 @@ top: vd = NULL; } - if (priority == ZIO_PRIORITY_ASYNC_READ) + /* + * We count both async reads and scrub IOs as asynchronous so + * that both can be upgraded in the event of a cache hit while + * the read IO is still in-flight. + */ + if (priority == ZIO_PRIORITY_ASYNC_READ || + priority == ZIO_PRIORITY_SCRUB) arc_hdr_set_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ); else arc_hdr_clear_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ); - if (hash_lock != NULL) - mutex_exit(hash_lock); - /* * At this point, we have a level 1 cache miss. Try again in * L2ARC if possible. @@ -5637,6 +5674,11 @@ top: ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY, B_FALSE); + acb->acb_zio_head = rzio; + + if (hash_lock != NULL) + mutex_exit(hash_lock); + DTRACE_PROBE2(l2arc__read, vdev_t *, vd, zio_t *, rzio); ARCSTAT_INCR(arcstat_l2_read_bytes, size); @@ -5651,6 +5693,8 @@ top: return (0); /* l2arc read error; goto zio_read() */ + if (hash_lock != NULL) + mutex_enter(hash_lock); } else { DTRACE_PROBE1(l2arc__miss, arc_buf_hdr_t *, hdr); @@ -5671,7 +5715,11 @@ top: rzio = zio_read(pio, spa, bp, hdr->b_l1hdr.b_pabd, size, arc_read_done, hdr, priority, zio_flags, zb); + acb->acb_zio_head = rzio; + if (hash_lock != NULL) + mutex_exit(hash_lock); + if (*arc_flags & ARC_FLAG_WAIT) return (zio_wait(rzio)); @@ -6162,9 +6210,9 @@ arc_write_done(zio_t *zio) zio_t * arc_write(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, arc_buf_t *buf, - boolean_t l2arc, const zio_prop_t *zp, arc_done_func_t *ready, - arc_done_func_t *children_ready, arc_done_func_t *physdone, - arc_done_func_t *done, void *private, zio_priority_t priority, + boolean_t l2arc, const zio_prop_t *zp, arc_write_done_func_t *ready, + arc_write_done_func_t *children_ready, arc_write_done_func_t *physdone, + arc_write_done_func_t *done, void *private, zio_priority_t priority, int zio_flags, const zbookmark_phys_t *zb) { arc_buf_hdr_t *hdr = buf->b_hdr; @@ -6590,9 +6638,6 @@ arc_init(void) mutex_init(&arc_dnlc_evicts_lock, NULL, MUTEX_DEFAULT, NULL); cv_init(&arc_dnlc_evicts_cv, NULL, CV_DEFAULT, NULL); - - /* Convert seconds to clock ticks */ - arc_min_prefetch_lifespan = 1 * hz; /* set min cache to 1/32 of all memory, or arc_abs_min, whichever is more */ arc_c_min = MAX(allmem / 32, arc_abs_min); Modified: projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c ============================================================================== --- projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c Sat Jun 9 23:38:22 2018 (r334904) +++ projects/pnfs-planb-server/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dbuf.c Sat Jun 9 23:45:05 2018 (r334905) @@ -902,7 +902,8 @@ dbuf_whichblock(dnode_t *dn, int64_t level, uint64_t o } static void -dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb) +dbuf_read_done(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp, + arc_buf_t *buf, void *vdb) { dmu_buf_impl_t *db = vdb; @@ -916,19 +917,22 @@ dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb) ASSERT(db->db.db_data == NULL); if (db->db_level == 0 && db->db_freed_in_flight) { /* we were freed in flight; disregard any error */ + if (buf == NULL) { + buf = arc_alloc_buf(db->db_objset->os_spa, + db, DBUF_GET_BUFC_TYPE(db), db->db.db_size); + } arc_release(buf, db); bzero(buf->b_data, db->db.db_size); arc_buf_freeze(buf); db->db_freed_in_flight = FALSE; dbuf_set_data(db, buf); db->db_state = DB_CACHED; - } else if (zio == NULL || zio->io_error == 0) { *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201806092345.w59Nj6QL002170>