Date: Sun, 2 Feb 2020 12:54:39 +0000 (UTC) From: Dimitry Andric <dim@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r357408 - in projects/clang1000-import: . share/man/man4 share/man/man9 sys/arm/arm sys/arm/conf sys/arm/include sys/arm/mv sys/arm/mv/discovery sys/arm/mv/kirkwood sys/arm/mv/orion sys... Message-ID: <202002021254.012Csd3q056280@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: dim Date: Sun Feb 2 12:54:38 2020 New Revision: 357408 URL: https://svnweb.freebsd.org/changeset/base/357408 Log: Merge ^/head r357389 through r357407. Added: projects/clang1000-import/sys/arm/arm/busdma_machdep.c - copied unchanged from r357407, head/sys/arm/arm/busdma_machdep.c Deleted: projects/clang1000-import/share/man/man4/vpo.4 projects/clang1000-import/sys/arm/arm/busdma_machdep-v4.c projects/clang1000-import/sys/arm/arm/busdma_machdep-v6.c projects/clang1000-import/sys/arm/conf/DB-78XXX projects/clang1000-import/sys/arm/conf/DB-88F5XXX projects/clang1000-import/sys/arm/conf/DB-88F6XXX projects/clang1000-import/sys/arm/conf/DOCKSTAR projects/clang1000-import/sys/arm/conf/DREAMPLUG-1001 projects/clang1000-import/sys/arm/conf/NOTES.armv7 projects/clang1000-import/sys/arm/conf/RT1310 projects/clang1000-import/sys/arm/conf/SHEEVAPLUG projects/clang1000-import/sys/arm/conf/TS7800 projects/clang1000-import/sys/arm/conf/genboardid.awk projects/clang1000-import/sys/arm/conf/mach-types projects/clang1000-import/sys/arm/conf/std.arm projects/clang1000-import/sys/arm/include/board.h projects/clang1000-import/sys/arm/mv/armv5_machdep.c projects/clang1000-import/sys/arm/mv/discovery/discovery.c projects/clang1000-import/sys/arm/mv/discovery/files.db78xxx projects/clang1000-import/sys/arm/mv/discovery/std.db78xxx projects/clang1000-import/sys/arm/mv/files.mv projects/clang1000-import/sys/arm/mv/kirkwood/files.kirkwood projects/clang1000-import/sys/arm/mv/kirkwood/kirkwood.c projects/clang1000-import/sys/arm/mv/kirkwood/std.db88f6xxx projects/clang1000-import/sys/arm/mv/kirkwood/std.kirkwood projects/clang1000-import/sys/arm/mv/orion/db88f5xxx.c projects/clang1000-import/sys/arm/mv/orion/files.db88f5xxx projects/clang1000-import/sys/arm/mv/orion/files.ts7800 projects/clang1000-import/sys/arm/mv/orion/orion.c projects/clang1000-import/sys/arm/mv/orion/std.db88f5xxx projects/clang1000-import/sys/arm/mv/orion/std.ts7800 projects/clang1000-import/sys/arm/ralink/files.ralink projects/clang1000-import/sys/arm/ralink/if_fv.c projects/clang1000-import/sys/arm/ralink/if_fvreg.h projects/clang1000-import/sys/arm/ralink/rt1310_gpio.c projects/clang1000-import/sys/arm/ralink/rt1310_intc.c projects/clang1000-import/sys/arm/ralink/rt1310_machdep.c projects/clang1000-import/sys/arm/ralink/rt1310_timer.c projects/clang1000-import/sys/arm/ralink/rt1310reg.h projects/clang1000-import/sys/arm/ralink/rt1310var.h projects/clang1000-import/sys/arm/ralink/std.ralink projects/clang1000-import/sys/dev/ppbus/immio.c projects/clang1000-import/sys/dev/ppbus/vpo.c projects/clang1000-import/sys/dev/ppbus/vpoio.c projects/clang1000-import/sys/dev/ppbus/vpoio.h projects/clang1000-import/sys/modules/vpo/Makefile Modified: projects/clang1000-import/Makefile projects/clang1000-import/Makefile.inc1 projects/clang1000-import/ObsoleteFiles.inc projects/clang1000-import/share/man/man4/Makefile projects/clang1000-import/share/man/man4/ppbus.4 projects/clang1000-import/share/man/man9/microseq.9 projects/clang1000-import/sys/arm/conf/NOTES projects/clang1000-import/sys/conf/NOTES projects/clang1000-import/sys/conf/files projects/clang1000-import/sys/conf/files.amd64 projects/clang1000-import/sys/conf/files.arm projects/clang1000-import/sys/conf/files.i386 projects/clang1000-import/sys/conf/files.powerpc projects/clang1000-import/sys/conf/files.x86 projects/clang1000-import/sys/conf/kern.pre.mk projects/clang1000-import/sys/conf/makeLINT.mk projects/clang1000-import/sys/conf/options projects/clang1000-import/sys/conf/options.arm projects/clang1000-import/sys/dev/tpm/tpm_crb.c projects/clang1000-import/sys/fs/nullfs/null_vnops.c projects/clang1000-import/sys/kern/kern_descrip.c projects/clang1000-import/sys/kern/vfs_subr.c projects/clang1000-import/sys/kern/vnode_if.src projects/clang1000-import/sys/modules/Makefile projects/clang1000-import/sys/sys/vnode.h projects/clang1000-import/sys/vm/uma_core.c projects/clang1000-import/sys/x86/iommu/intel_gas.c Directory Properties: projects/clang1000-import/ (props changed) Modified: projects/clang1000-import/Makefile ============================================================================== --- projects/clang1000-import/Makefile Sun Feb 2 11:37:27 2020 (r357407) +++ projects/clang1000-import/Makefile Sun Feb 2 12:54:38 2020 (r357408) @@ -89,7 +89,7 @@ # # See src/UPDATING `COMMON ITEMS' for more complete information. # -# If TARGET=machine (e.g. powerpc, sparc64, ...) is specified you can +# If TARGET=machine (e.g. powerpc, arm64, ...) is specified you can # cross build world for other machine types using the buildworld target, # and once the world is built you can cross build a kernel using the # buildkernel target. @@ -488,7 +488,7 @@ worlds: .PHONY # In all cases, if the user specifies TARGETS on the command line, # honor that most of all. # -TARGETS?=amd64 arm arm64 i386 mips powerpc riscv sparc64 +TARGETS?=amd64 arm arm64 i386 mips powerpc riscv _UNIVERSE_TARGETS= ${TARGETS} TARGET_ARCHES_arm?= armv6 armv7 TARGET_ARCHES_arm64?= aarch64 @@ -501,14 +501,12 @@ TARGET_ARCHES_${target}?= ${target} .endfor MAKE_PARAMS_mips?= CROSS_TOOLCHAIN=mips-gcc6 -MAKE_PARAMS_sparc64?= CROSS_TOOLCHAIN=sparc64-gcc6 TOOLCHAINS_mips= mips-gcc6 -TOOLCHAINS_sparc64= sparc64-gcc6 # Remove architectures only supported by external toolchain from # universe if required toolchain packages are missing. -.for target in mips sparc64 +.for target in mips .if ${_UNIVERSE_TARGETS:M${target}} .for toolchain in ${TOOLCHAINS_${target}} .if !exists(/usr/local/share/toolchains/${toolchain}.mk) Modified: projects/clang1000-import/Makefile.inc1 ============================================================================== --- projects/clang1000-import/Makefile.inc1 Sun Feb 2 11:37:27 2020 (r357407) +++ projects/clang1000-import/Makefile.inc1 Sun Feb 2 12:54:38 2020 (r357408) @@ -153,8 +153,7 @@ KNOWN_ARCHES?= aarch64/arm64 \ powerpc64/powerpc \ powerpcspe/powerpc \ riscv64/riscv \ - riscv64sf/riscv \ - sparc64 + riscv64sf/riscv .if ${TARGET} == ${TARGET_ARCH} _t= ${TARGET} @@ -2290,8 +2289,6 @@ _basic_bootstrap_tools_multilink+=bin/test test,[ _basic_bootstrap_tools=usr.bin/awk usr.bin/cut bin/expr usr.bin/gencat \ usr.bin/join usr.bin/mktemp bin/rmdir usr.bin/sed usr.bin/sort \ usr.bin/truncate usr.bin/tsort -# elf2aout is required for sparc64 build -_basic_bootstrap_tools+=usr.bin/elf2aout # file2c is required for building usr.sbin/config: _basic_bootstrap_tools+=usr.bin/file2c # uuencode/uudecode required for share/tabset @@ -2915,14 +2912,10 @@ _cddl_lib_libctf= cddl/lib/libctf _cddl_lib= cddl/lib cddl/lib/libctf__L: lib/libz__L .endif -# cddl/lib/libdtrace requires lib/libproc and lib/librtld_db; it's only built -# on select architectures though (see cddl/lib/Makefile) -.if ${MACHINE_CPUARCH} != "sparc64" _prebuild_libs+= lib/libprocstat lib/libproc lib/librtld_db lib/libprocstat__L: lib/libelf__L lib/libkvm__L lib/libutil__L lib/libproc__L: lib/libprocstat__L lib/librtld_db__L: lib/libprocstat__L -.endif .if ${MK_CRYPT} != "no" .if ${MK_OPENSSL} != "no" Modified: projects/clang1000-import/ObsoleteFiles.inc ============================================================================== --- projects/clang1000-import/ObsoleteFiles.inc Sun Feb 2 11:37:27 2020 (r357407) +++ projects/clang1000-import/ObsoleteFiles.inc Sun Feb 2 12:54:38 2020 (r357408) @@ -272,6 +272,10 @@ OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt OLD_DIRS+=usr/lib/clang/9.0.1/lib/freebsd OLD_DIRS+=usr/lib/clang/9.0.1/lib OLD_DIRS+=usr/lib/clang/9.0.1 +# 20200127: vpo removed +OLD_FILES+=usr/share/man/man4/imm.4.gz +OLD_FILES+=usr/share/man/man4/vpo.4.gz + # 20200104: gcc libssp removed OLD_FILES+=usr/include/ssp/ssp.h OLD_FILES+=usr/include/ssp/stdio.h Modified: projects/clang1000-import/share/man/man4/Makefile ============================================================================== --- projects/clang1000-import/share/man/man4/Makefile Sun Feb 2 11:37:27 2020 (r357407) +++ projects/clang1000-import/share/man/man4/Makefile Sun Feb 2 12:54:38 2020 (r357408) @@ -553,7 +553,6 @@ MAN= aac.4 \ ${_vmd.4} \ ${_vmm.4} \ ${_vmx.4} \ - vpo.4 \ vr.4 \ vt.4 \ vte.4 \ @@ -740,7 +739,6 @@ MLINKS+=vge.4 if_vge.4 MLINKS+=vlan.4 if_vlan.4 MLINKS+=vxlan.4 if_vxlan.4 MLINKS+=${_vmx.4} ${_if_vmx.4} -MLINKS+=vpo.4 imm.4 MLINKS+=vr.4 if_vr.4 MLINKS+=vte.4 if_vte.4 MLINKS+=${_vtnet.4} ${_if_vtnet.4} Modified: projects/clang1000-import/share/man/man4/ppbus.4 ============================================================================== --- projects/clang1000-import/share/man/man4/ppbus.4 Sun Feb 2 11:37:27 2020 (r357407) +++ projects/clang1000-import/share/man/man4/ppbus.4 Sun Feb 2 12:54:38 2020 (r357408) @@ -33,8 +33,6 @@ .Sh SYNOPSIS .Cd "device ppbus" .Pp -.Cd "device vpo" -.Pp .Cd "device lpt" .Cd "device plip" .Cd "device ppi" @@ -66,8 +64,6 @@ and non-standard software: .Pp .Bl -column "Driver" -compact .It Em Driver Ta Em Description -.It Sy vpo Ta "VPI0 parallel to Adaptec AIC-7110 SCSI controller driver" . -It uses standard and non-standard parallel port accesses. .It Sy ppi Ta "Parallel port interface for general I/O" .It Sy pps Ta "Pulse per second Timing Interface" .It Sy lpbb Ta "Philips official parallel port I2C bit-banging interface" @@ -336,22 +332,11 @@ operation (opcodes are described in .Xr microseq 9 ) . Standard I/O operations are implemented at ppbus level whereas basic I/O operations and microseq language are coded at adapter level for efficiency. -.Pp -As an example, the -.Xr vpo 4 -driver uses microsequences to implement: -.Bl -bullet -offset indent -.It -a modified version of the NIBBLE transfer mode -.It -various I/O sequences to initialize, select and allocate the peripheral -.El .Sh SEE ALSO .Xr lpt 4 , .Xr plip 4 , .Xr ppc 4 , .Xr ppi 4 , -.Xr vpo 4 .Sh HISTORY The .Nm Modified: projects/clang1000-import/share/man/man9/microseq.9 ============================================================================== --- projects/clang1000-import/share/man/man9/microseq.9 Sun Feb 2 11:37:27 2020 (r357407) +++ projects/clang1000-import/share/man/man9/microseq.9 Sun Feb 2 12:54:38 2020 (r357408) @@ -51,7 +51,7 @@ efficient code Before using microsequences, you are encouraged to look at .Xr ppc 4 microsequencer implementation and an example of how using it in -.Xr vpo 4 . +.Xr ppi 4 . .Sh PPBUS register model .Ss Background The parallel port model chosen for ppbus is the PC parallel port model. @@ -477,7 +477,7 @@ executed at ppbus layer. .Sh SEE ALSO .Xr ppbus 4 , .Xr ppc 4 , -.Xr vpo 4 +.Xr ppi 4 .Sh HISTORY The .Nm Copied: projects/clang1000-import/sys/arm/arm/busdma_machdep.c (from r357407, head/sys/arm/arm/busdma_machdep.c) ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/clang1000-import/sys/arm/arm/busdma_machdep.c Sun Feb 2 12:54:38 2020 (r357408, copy of r357407, head/sys/arm/arm/busdma_machdep.c) @@ -0,0 +1,1784 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2012-2015 Ian Lepore + * Copyright (c) 2010 Mark Tinguely + * Copyright (c) 2004 Olivier Houchard + * Copyright (c) 2002 Peter Grehan + * Copyright (c) 1997, 1998 Justin T. Gibbs. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions, and the following disclaimer, + * without modification, immediately at the beginning of the file. + * 2. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * From i386/busdma_machdep.c 191438 2009-04-23 20:24:19Z jhb + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/malloc.h> +#include <sys/bus.h> +#include <sys/busdma_bufalloc.h> +#include <sys/counter.h> +#include <sys/interrupt.h> +#include <sys/kernel.h> +#include <sys/ktr.h> +#include <sys/lock.h> +#include <sys/memdesc.h> +#include <sys/proc.h> +#include <sys/mutex.h> +#include <sys/sysctl.h> +#include <sys/uio.h> + +#include <vm/vm.h> +#include <vm/vm_param.h> +#include <vm/vm_page.h> +#include <vm/vm_phys.h> +#include <vm/vm_map.h> +#include <vm/vm_extern.h> +#include <vm/vm_kern.h> + +#include <machine/atomic.h> +#include <machine/bus.h> +#include <machine/cpu.h> +#include <machine/md_var.h> + +#define BUSDMA_DCACHE_ALIGN cpuinfo.dcache_line_size +#define BUSDMA_DCACHE_MASK cpuinfo.dcache_line_mask + +#define MAX_BPAGES 64 +#define MAX_DMA_SEGMENTS 4096 +#define BUS_DMA_EXCL_BOUNCE BUS_DMA_BUS2 +#define BUS_DMA_ALIGN_BOUNCE BUS_DMA_BUS3 +#define BUS_DMA_COULD_BOUNCE (BUS_DMA_EXCL_BOUNCE | BUS_DMA_ALIGN_BOUNCE) +#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 + +struct bounce_zone; + +struct bus_dma_tag { + bus_dma_tag_t parent; + bus_size_t alignment; + bus_addr_t boundary; + bus_addr_t lowaddr; + bus_addr_t highaddr; + bus_dma_filter_t *filter; + void *filterarg; + bus_size_t maxsize; + u_int nsegments; + bus_size_t maxsegsz; + int flags; + int ref_count; + int map_count; + bus_dma_lock_t *lockfunc; + void *lockfuncarg; + struct bounce_zone *bounce_zone; +}; + +struct bounce_page { + vm_offset_t vaddr; /* kva of bounce buffer */ + bus_addr_t busaddr; /* Physical address */ + vm_offset_t datavaddr; /* kva of client data */ + vm_page_t datapage; /* physical page of client data */ + vm_offset_t dataoffs; /* page offset of client data */ + bus_size_t datacount; /* client data count */ + STAILQ_ENTRY(bounce_page) links; +}; + +struct sync_list { + vm_offset_t vaddr; /* kva of client data */ + bus_addr_t paddr; /* physical address */ + vm_page_t pages; /* starting page of client data */ + bus_size_t datacount; /* client data count */ +}; + +int busdma_swi_pending; + +struct bounce_zone { + STAILQ_ENTRY(bounce_zone) links; + STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; + int total_bpages; + int free_bpages; + int reserved_bpages; + int active_bpages; + int total_bounced; + int total_deferred; + int map_count; + bus_size_t alignment; + bus_addr_t lowaddr; + char zoneid[8]; + char lowaddrid[20]; + struct sysctl_ctx_list sysctl_tree; + struct sysctl_oid *sysctl_tree_top; +}; + +static struct mtx bounce_lock; +static int total_bpages; +static int busdma_zonecount; +static uint32_t tags_total; +static uint32_t maps_total; +static uint32_t maps_dmamem; +static uint32_t maps_coherent; +static counter_u64_t maploads_total; +static counter_u64_t maploads_bounced; +static counter_u64_t maploads_coherent; +static counter_u64_t maploads_dmamem; +static counter_u64_t maploads_mbuf; +static counter_u64_t maploads_physmem; + +static STAILQ_HEAD(, bounce_zone) bounce_zone_list; + +SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters"); +SYSCTL_UINT(_hw_busdma, OID_AUTO, tags_total, CTLFLAG_RD, &tags_total, 0, + "Number of active tags"); +SYSCTL_UINT(_hw_busdma, OID_AUTO, maps_total, CTLFLAG_RD, &maps_total, 0, + "Number of active maps"); +SYSCTL_UINT(_hw_busdma, OID_AUTO, maps_dmamem, CTLFLAG_RD, &maps_dmamem, 0, + "Number of active maps for bus_dmamem_alloc buffers"); +SYSCTL_UINT(_hw_busdma, OID_AUTO, maps_coherent, CTLFLAG_RD, &maps_coherent, 0, + "Number of active maps with BUS_DMA_COHERENT flag set"); +SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_total, CTLFLAG_RD, + &maploads_total, "Number of load operations performed"); +SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_bounced, CTLFLAG_RD, + &maploads_bounced, "Number of load operations that used bounce buffers"); +SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_coherent, CTLFLAG_RD, + &maploads_dmamem, "Number of load operations on BUS_DMA_COHERENT memory"); +SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_dmamem, CTLFLAG_RD, + &maploads_dmamem, "Number of load operations on bus_dmamem_alloc buffers"); +SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_mbuf, CTLFLAG_RD, + &maploads_mbuf, "Number of load operations for mbufs"); +SYSCTL_COUNTER_U64(_hw_busdma, OID_AUTO, maploads_physmem, CTLFLAG_RD, + &maploads_physmem, "Number of load operations on physical buffers"); +SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0, + "Total bounce pages"); + +struct bus_dmamap { + struct bp_list bpages; + int pagesneeded; + int pagesreserved; + bus_dma_tag_t dmat; + struct memdesc mem; + bus_dmamap_callback_t *callback; + void *callback_arg; + int flags; +#define DMAMAP_COHERENT (1 << 0) +#define DMAMAP_DMAMEM_ALLOC (1 << 1) +#define DMAMAP_MBUF (1 << 2) + STAILQ_ENTRY(bus_dmamap) links; + bus_dma_segment_t *segments; + int sync_count; + struct sync_list slist[]; +}; + +static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; +static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; + +static void init_bounce_pages(void *dummy); +static int alloc_bounce_zone(bus_dma_tag_t dmat); +static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); +static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, + int commit); +static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, + vm_offset_t vaddr, bus_addr_t addr, bus_size_t size); +static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); +static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, pmap_t pmap, + bus_dmamap_t map, void *buf, bus_size_t buflen, int flags); +static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, + vm_paddr_t buf, bus_size_t buflen, int flags); +static int _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, + int flags); +static void dma_preread_safe(vm_offset_t va, vm_paddr_t pa, vm_size_t size); +static void dma_dcache_sync(struct sync_list *sl, bus_dmasync_op_t op); + +static busdma_bufalloc_t coherent_allocator; /* Cache of coherent buffers */ +static busdma_bufalloc_t standard_allocator; /* Cache of standard buffers */ + +MALLOC_DEFINE(M_BUSDMA, "busdma", "busdma metadata"); +MALLOC_DEFINE(M_BOUNCE, "bounce", "busdma bounce pages"); + +static void +busdma_init(void *dummy) +{ + int uma_flags; + + maploads_total = counter_u64_alloc(M_WAITOK); + maploads_bounced = counter_u64_alloc(M_WAITOK); + maploads_coherent = counter_u64_alloc(M_WAITOK); + maploads_dmamem = counter_u64_alloc(M_WAITOK); + maploads_mbuf = counter_u64_alloc(M_WAITOK); + maploads_physmem = counter_u64_alloc(M_WAITOK); + + uma_flags = 0; + + /* Create a cache of buffers in standard (cacheable) memory. */ + standard_allocator = busdma_bufalloc_create("buffer", + BUSDMA_DCACHE_ALIGN,/* minimum_alignment */ + NULL, /* uma_alloc func */ + NULL, /* uma_free func */ + uma_flags); /* uma_zcreate_flags */ + +#ifdef INVARIANTS + /* + * Force UMA zone to allocate service structures like + * slabs using own allocator. uma_debug code performs + * atomic ops on uma_slab_t fields and safety of this + * operation is not guaranteed for write-back caches + */ + uma_flags = UMA_ZONE_NOTOUCH; +#endif + /* + * Create a cache of buffers in uncacheable memory, to implement the + * BUS_DMA_COHERENT (and potentially BUS_DMA_NOCACHE) flag. + */ + coherent_allocator = busdma_bufalloc_create("coherent", + BUSDMA_DCACHE_ALIGN,/* minimum_alignment */ + busdma_bufalloc_alloc_uncacheable, + busdma_bufalloc_free_uncacheable, + uma_flags); /* uma_zcreate_flags */ +} + +/* + * This init historically used SI_SUB_VM, but now the init code requires + * malloc(9) using M_BUSDMA memory and the pcpu zones for counter(9), which get + * set up by SI_SUB_KMEM and SI_ORDER_LAST, so we'll go right after that by + * using SI_SUB_KMEM+1. + */ +SYSINIT(busdma, SI_SUB_KMEM+1, SI_ORDER_FIRST, busdma_init, NULL); + +/* + * This routine checks the exclusion zone constraints from a tag against the + * physical RAM available on the machine. If a tag specifies an exclusion zone + * but there's no RAM in that zone, then we avoid allocating resources to bounce + * a request, and we can use any memory allocator (as opposed to needing + * kmem_alloc_contig() just because it can allocate pages in an address range). + * + * Most tags have BUS_SPACE_MAXADDR or BUS_SPACE_MAXADDR_32BIT (they are the + * same value on 32-bit architectures) as their lowaddr constraint, and we can't + * possibly have RAM at an address higher than the highest address we can + * express, so we take a fast out. + */ +static int +exclusion_bounce_check(vm_offset_t lowaddr, vm_offset_t highaddr) +{ + int i; + + if (lowaddr >= BUS_SPACE_MAXADDR) + return (0); + + for (i = 0; phys_avail[i] && phys_avail[i + 1]; i += 2) { + if ((lowaddr >= phys_avail[i] && lowaddr < phys_avail[i + 1]) || + (lowaddr < phys_avail[i] && highaddr >= phys_avail[i])) + return (1); + } + return (0); +} + +/* + * Return true if the tag has an exclusion zone that could lead to bouncing. + */ +static __inline int +exclusion_bounce(bus_dma_tag_t dmat) +{ + + return (dmat->flags & BUS_DMA_EXCL_BOUNCE); +} + +/* + * Return true if the given address does not fall on the alignment boundary. + */ +static __inline int +alignment_bounce(bus_dma_tag_t dmat, bus_addr_t addr) +{ + + return (addr & (dmat->alignment - 1)); +} + +/* + * Return true if the DMA should bounce because the start or end does not fall + * on a cacheline boundary (which would require a partial cacheline flush). + * COHERENT memory doesn't trigger cacheline flushes. Memory allocated by + * bus_dmamem_alloc() is always aligned to cacheline boundaries, and there's a + * strict rule that such memory cannot be accessed by the CPU while DMA is in + * progress (or by multiple DMA engines at once), so that it's always safe to do + * full cacheline flushes even if that affects memory outside the range of a + * given DMA operation that doesn't involve the full allocated buffer. If we're + * mapping an mbuf, that follows the same rules as a buffer we allocated. + */ +static __inline int +cacheline_bounce(bus_dmamap_t map, bus_addr_t addr, bus_size_t size) +{ + + if (map->flags & (DMAMAP_DMAMEM_ALLOC | DMAMAP_COHERENT | DMAMAP_MBUF)) + return (0); + return ((addr | size) & BUSDMA_DCACHE_MASK); +} + +/* + * Return true if we might need to bounce the DMA described by addr and size. + * + * This is used to quick-check whether we need to do the more expensive work of + * checking the DMA page-by-page looking for alignment and exclusion bounces. + * + * Note that the addr argument might be either virtual or physical. It doesn't + * matter because we only look at the low-order bits, which are the same in both + * address spaces and maximum alignment of generic buffer is limited up to page + * size. + * Bouncing of buffers allocated by bus_dmamem_alloc()is not necessary, these + * always comply with the required rules (alignment, boundary, and address + * range). + */ +static __inline int +might_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t addr, + bus_size_t size) +{ + + KASSERT(map->flags & DMAMAP_DMAMEM_ALLOC || + dmat->alignment <= PAGE_SIZE, + ("%s: unsupported alignment (0x%08lx) for buffer not " + "allocated by bus_dmamem_alloc()", + __func__, dmat->alignment)); + + return (!(map->flags & DMAMAP_DMAMEM_ALLOC) && + ((dmat->flags & BUS_DMA_EXCL_BOUNCE) || + alignment_bounce(dmat, addr) || + cacheline_bounce(map, addr, size))); +} + +/* + * Return true if we must bounce the DMA described by paddr and size. + * + * Bouncing can be triggered by DMA that doesn't begin and end on cacheline + * boundaries, or doesn't begin on an alignment boundary, or falls within the + * exclusion zone of any tag in the ancestry chain. + * + * For exclusions, walk the chain of tags comparing paddr to the exclusion zone + * within each tag. If the tag has a filter function, use it to decide whether + * the DMA needs to bounce, otherwise any DMA within the zone bounces. + */ +static int +must_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t paddr, + bus_size_t size) +{ + + if (cacheline_bounce(map, paddr, size)) + return (1); + + /* + * The tag already contains ancestors' alignment restrictions so this + * check doesn't need to be inside the loop. + */ + if (alignment_bounce(dmat, paddr)) + return (1); + + /* + * Even though each tag has an exclusion zone that is a superset of its + * own and all its ancestors' exclusions, the exclusion zone of each tag + * up the chain must be checked within the loop, because the busdma + * rules say the filter function is called only when the address lies + * within the low-highaddr range of the tag that filterfunc belongs to. + */ + while (dmat != NULL && exclusion_bounce(dmat)) { + if ((paddr >= dmat->lowaddr && paddr <= dmat->highaddr) && + (dmat->filter == NULL || + dmat->filter(dmat->filterarg, paddr) != 0)) + return (1); + dmat = dmat->parent; + } + + return (0); +} + +/* + * Convenience function for manipulating driver locks from busdma (during + * busdma_swi, for example). Drivers that don't provide their own locks + * should specify &Giant to dmat->lockfuncarg. Drivers that use their own + * non-mutex locking scheme don't have to use this at all. + */ +void +busdma_lock_mutex(void *arg, bus_dma_lock_op_t op) +{ + struct mtx *dmtx; + + dmtx = (struct mtx *)arg; + switch (op) { + case BUS_DMA_LOCK: + mtx_lock(dmtx); + break; + case BUS_DMA_UNLOCK: + mtx_unlock(dmtx); + break; + default: + panic("Unknown operation 0x%x for busdma_lock_mutex!", op); + } +} + +/* + * dflt_lock should never get called. It gets put into the dma tag when + * lockfunc == NULL, which is only valid if the maps that are associated + * with the tag are meant to never be defered. + * XXX Should have a way to identify which driver is responsible here. + */ +static void +dflt_lock(void *arg, bus_dma_lock_op_t op) +{ + + panic("driver error: busdma dflt_lock called"); +} + +/* + * Allocate a device specific dma_tag. + */ +int +bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, + bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, + bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, + int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, + void *lockfuncarg, bus_dma_tag_t *dmat) +{ + bus_dma_tag_t newtag; + int error = 0; + + /* Basic sanity checking. */ + KASSERT(boundary == 0 || powerof2(boundary), + ("dma tag boundary %lu, must be a power of 2", boundary)); + KASSERT(boundary == 0 || boundary >= maxsegsz, + ("dma tag boundary %lu is < maxsegsz %lu\n", boundary, maxsegsz)); + KASSERT(alignment != 0 && powerof2(alignment), + ("dma tag alignment %lu, must be non-zero power of 2", alignment)); + KASSERT(maxsegsz != 0, ("dma tag maxsegsz must not be zero")); + + /* Return a NULL tag on failure */ + *dmat = NULL; + + newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_BUSDMA, + M_ZERO | M_NOWAIT); + if (newtag == NULL) { + CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", + __func__, newtag, 0, error); + return (ENOMEM); + } + + newtag->parent = parent; + newtag->alignment = alignment; + newtag->boundary = boundary; + newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1); + newtag->highaddr = trunc_page((vm_paddr_t)highaddr) + + (PAGE_SIZE - 1); + newtag->filter = filter; + newtag->filterarg = filterarg; + newtag->maxsize = maxsize; + newtag->nsegments = nsegments; + newtag->maxsegsz = maxsegsz; + newtag->flags = flags; + newtag->ref_count = 1; /* Count ourself */ + newtag->map_count = 0; + if (lockfunc != NULL) { + newtag->lockfunc = lockfunc; + newtag->lockfuncarg = lockfuncarg; + } else { + newtag->lockfunc = dflt_lock; + newtag->lockfuncarg = NULL; + } + + /* Take into account any restrictions imposed by our parent tag */ + if (parent != NULL) { + newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); + newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); + newtag->alignment = MAX(parent->alignment, newtag->alignment); + newtag->flags |= parent->flags & BUS_DMA_COULD_BOUNCE; + newtag->flags |= parent->flags & BUS_DMA_COHERENT; + if (newtag->boundary == 0) + newtag->boundary = parent->boundary; + else if (parent->boundary != 0) + newtag->boundary = MIN(parent->boundary, + newtag->boundary); + if (newtag->filter == NULL) { + /* + * Short circuit to looking at our parent directly + * since we have encapsulated all of its information + */ + newtag->filter = parent->filter; + newtag->filterarg = parent->filterarg; + newtag->parent = parent->parent; + } + if (newtag->parent != NULL) + atomic_add_int(&parent->ref_count, 1); + } + + if (exclusion_bounce_check(newtag->lowaddr, newtag->highaddr)) + newtag->flags |= BUS_DMA_EXCL_BOUNCE; + if (alignment_bounce(newtag, 1)) + newtag->flags |= BUS_DMA_ALIGN_BOUNCE; + + /* + * Any request can auto-bounce due to cacheline alignment, in addition + * to any alignment or boundary specifications in the tag, so if the + * ALLOCNOW flag is set, there's always work to do. + */ + if ((flags & BUS_DMA_ALLOCNOW) != 0) { + struct bounce_zone *bz; + /* + * Round size up to a full page, and add one more page because + * there can always be one more boundary crossing than the + * number of pages in a transfer. + */ + maxsize = roundup2(maxsize, PAGE_SIZE) + PAGE_SIZE; + + if ((error = alloc_bounce_zone(newtag)) != 0) { + free(newtag, M_BUSDMA); + return (error); + } + bz = newtag->bounce_zone; + + if (ptoa(bz->total_bpages) < maxsize) { + int pages; + + pages = atop(maxsize) - bz->total_bpages; + + /* Add pages to our bounce pool */ + if (alloc_bounce_pages(newtag, pages) < pages) + error = ENOMEM; + } + /* Performed initial allocation */ + newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; + } else + newtag->bounce_zone = NULL; + + if (error != 0) { + free(newtag, M_BUSDMA); + } else { + atomic_add_32(&tags_total, 1); + *dmat = newtag; + } + CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", + __func__, newtag, (newtag != NULL ? newtag->flags : 0), error); + return (error); +} + +void +bus_dma_template_init(bus_dma_tag_template_t *t, bus_dma_tag_t parent) +{ + + if (t == NULL) + return; + + t->parent = parent; + t->alignment = 1; + t->boundary = 0; + t->lowaddr = t->highaddr = BUS_SPACE_MAXADDR; + t->maxsize = t->maxsegsize = BUS_SPACE_MAXSIZE; + t->nsegments = BUS_SPACE_UNRESTRICTED; + t->lockfunc = NULL; + t->lockfuncarg = NULL; + t->flags = 0; +} + +int +bus_dma_template_tag(bus_dma_tag_template_t *t, bus_dma_tag_t *dmat) +{ + + if (t == NULL || dmat == NULL) + return (EINVAL); + + return (bus_dma_tag_create(t->parent, t->alignment, t->boundary, + t->lowaddr, t->highaddr, NULL, NULL, t->maxsize, + t->nsegments, t->maxsegsize, t->flags, t->lockfunc, t->lockfuncarg, + dmat)); +} + +void +bus_dma_template_clone(bus_dma_tag_template_t *t, bus_dma_tag_t dmat) +{ + + if (t == NULL || dmat == NULL) + return; + + t->parent = dmat->parent; + t->alignment = dmat->alignment; + t->boundary = dmat->boundary; + t->lowaddr = dmat->lowaddr; + t->highaddr = dmat->highaddr; + t->maxsize = dmat->maxsize; + t->nsegments = dmat->nsegments; + t->maxsegsize = dmat->maxsegsz; + t->flags = dmat->flags; + t->lockfunc = dmat->lockfunc; + t->lockfuncarg = dmat->lockfuncarg; +} + +int +bus_dma_tag_set_domain(bus_dma_tag_t dmat, int domain) +{ + + return (0); +} + +int +bus_dma_tag_destroy(bus_dma_tag_t dmat) +{ + bus_dma_tag_t dmat_copy; + int error; + + error = 0; + dmat_copy = dmat; + + if (dmat != NULL) { + + if (dmat->map_count != 0) { + error = EBUSY; + goto out; + } + + while (dmat != NULL) { + bus_dma_tag_t parent; + + parent = dmat->parent; + atomic_subtract_int(&dmat->ref_count, 1); + if (dmat->ref_count == 0) { + atomic_subtract_32(&tags_total, 1); + free(dmat, M_BUSDMA); + /* + * Last reference count, so + * release our reference + * count on our parent. + */ + dmat = parent; + } else + dmat = NULL; + } + } +out: + CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error); + return (error); +} + +static int +allocate_bz_and_pages(bus_dma_tag_t dmat, bus_dmamap_t mapp) +{ + struct bounce_zone *bz; + int maxpages; + int error; + + if (dmat->bounce_zone == NULL) + if ((error = alloc_bounce_zone(dmat)) != 0) + return (error); + bz = dmat->bounce_zone; + /* Initialize the new map */ + STAILQ_INIT(&(mapp->bpages)); + + /* + * Attempt to add pages to our pool on a per-instance basis up to a sane + * limit. Even if the tag isn't flagged as COULD_BOUNCE due to + * alignment and boundary constraints, it could still auto-bounce due to + * cacheline alignment, which requires at most two bounce pages. + */ + if (dmat->flags & BUS_DMA_COULD_BOUNCE) + maxpages = MAX_BPAGES; + else + maxpages = 2 * bz->map_count; + if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 || + (bz->map_count > 0 && bz->total_bpages < maxpages)) { + int pages; + + pages = atop(roundup2(dmat->maxsize, PAGE_SIZE)) + 1; + pages = MIN(maxpages - bz->total_bpages, pages); + pages = MAX(pages, 2); + if (alloc_bounce_pages(dmat, pages) < pages) + return (ENOMEM); + + if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) + dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; + } + bz->map_count++; + return (0); +} + +static bus_dmamap_t +allocate_map(bus_dma_tag_t dmat, int mflags) +{ + int mapsize, segsize; + bus_dmamap_t map; + + /* + * Allocate the map. The map structure ends with an embedded + * variable-sized array of sync_list structures. Following that + * we allocate enough extra space to hold the array of bus_dma_segments. + */ + KASSERT(dmat->nsegments <= MAX_DMA_SEGMENTS, + ("cannot allocate %u dma segments (max is %u)", + dmat->nsegments, MAX_DMA_SEGMENTS)); + segsize = sizeof(struct bus_dma_segment) * dmat->nsegments; + mapsize = sizeof(*map) + sizeof(struct sync_list) * dmat->nsegments; + map = malloc(mapsize + segsize, M_BUSDMA, mflags | M_ZERO); + if (map == NULL) { + CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM); + return (NULL); + } + map->segments = (bus_dma_segment_t *)((uintptr_t)map + mapsize); + STAILQ_INIT(&map->bpages); + return (map); +} + +/* + * Allocate a handle for mapping from kva/uva/physical + * address space into bus device space. + */ +int +bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) +{ + bus_dmamap_t map; + int error = 0; + + *mapp = map = allocate_map(dmat, M_NOWAIT); + if (map == NULL) { + CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, ENOMEM); + return (ENOMEM); + } + + /* + * Bouncing might be required if the driver asks for an exclusion + * region, a data alignment that is stricter than 1, or DMA that begins + * or ends with a partial cacheline. Whether bouncing will actually + * happen can't be known until mapping time, but we need to pre-allocate + * resources now because we might not be allowed to at mapping time. + */ + error = allocate_bz_and_pages(dmat, map); + if (error != 0) { + free(map, M_BUSDMA); + *mapp = NULL; + return (error); + } + if (map->flags & DMAMAP_COHERENT) + atomic_add_32(&maps_coherent, 1); + atomic_add_32(&maps_total, 1); + dmat->map_count++; + + return (0); +} + +/* + * Destroy a handle for mapping from kva/uva/physical + * address space into bus device space. + */ +int +bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) +{ + + if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) { + CTR3(KTR_BUSDMA, "%s: tag %p error %d", + __func__, dmat, EBUSY); + return (EBUSY); + } + if (dmat->bounce_zone) + dmat->bounce_zone->map_count--; + if (map->flags & DMAMAP_COHERENT) + atomic_subtract_32(&maps_coherent, 1); + atomic_subtract_32(&maps_total, 1); + free(map, M_BUSDMA); + dmat->map_count--; + CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); + return (0); +} + +/* + * Allocate a piece of memory that can be efficiently mapped into bus device + * space based on the constraints listed in the dma tag. Returns a pointer to + * the allocated memory, and a pointer to an associated bus_dmamap. + */ +int +bus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddr, int flags, + bus_dmamap_t *mapp) +{ + busdma_bufalloc_t ba; + struct busdma_bufzone *bufzone; + bus_dmamap_t map; + vm_memattr_t memattr; + int mflags; + + if (flags & BUS_DMA_NOWAIT) + mflags = M_NOWAIT; + else + mflags = M_WAITOK; + if (flags & BUS_DMA_ZERO) + mflags |= M_ZERO; + + *mapp = map = allocate_map(dmat, mflags); + if (map == NULL) { + CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", + __func__, dmat, dmat->flags, ENOMEM); + return (ENOMEM); + } + map->flags = DMAMAP_DMAMEM_ALLOC; *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202002021254.012Csd3q056280>