Skip site navigation (1)Skip section navigation (2)
Date:      Wed, 7 Aug 2013 15:44:58 +0000 (UTC)
From:      Olivier Houchard <cognet@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r254061 - head/sys/arm/arm
Message-ID:  <201308071544.r77FiwMK072982@svn.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: cognet
Date: Wed Aug  7 15:44:58 2013
New Revision: 254061
URL: http://svnweb.freebsd.org/changeset/base/254061

Log:
  Don't bother trying to work around buffers which are not aligned on a cache
  line boundary. It has never been 100% correct, and it can't work on SMP,
  because nothing prevents another core from accessing data from an unrelated
  buffer in the same cache line while we invalidated it. Just use bounce pages
  instead.
  
  Reviewed by:	ian
  Approved by:	mux (mentor) (implicit)

Modified:
  head/sys/arm/arm/busdma_machdep-v6.c

Modified: head/sys/arm/arm/busdma_machdep-v6.c
==============================================================================
--- head/sys/arm/arm/busdma_machdep-v6.c	Wed Aug  7 12:03:34 2013	(r254060)
+++ head/sys/arm/arm/busdma_machdep-v6.c	Wed Aug  7 15:44:58 2013	(r254061)
@@ -67,8 +67,6 @@ __FBSDID("$FreeBSD$");
 #define BUS_DMA_COULD_BOUNCE	BUS_DMA_BUS3
 #define BUS_DMA_MIN_ALLOC_COMP	BUS_DMA_BUS4
 
-#define FIX_DMAP_BUS_DMASYNC_POSTREAD
-
 struct bounce_zone;
 
 struct bus_dma_tag {
@@ -179,7 +177,7 @@ static bus_addr_t add_bounce_page(bus_dm
 				  vm_offset_t vaddr, bus_addr_t addr,
 				  bus_size_t size);
 static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage);
-int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr);
+int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr, bus_size_t size, int coherent);
 static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
     void *buf, bus_size_t buflen, int flags);
 static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map,
@@ -269,7 +267,7 @@ _bus_dma_inrange(struct arm32_dma_range 
  * to check for a match, if there is no filter callback then assume a match.
  */
 int
-run_filter(bus_dma_tag_t dmat, bus_addr_t paddr)
+run_filter(bus_dma_tag_t dmat, bus_addr_t paddr, bus_size_t size, int coherent)
 {
 	int retval;
 
@@ -277,7 +275,9 @@ run_filter(bus_dma_tag_t dmat, bus_addr_
 
 	do {
 		if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr)
-		 || ((paddr & (dmat->alignment - 1)) != 0))
+		 || ((paddr & (dmat->alignment - 1)) != 0) ||
+		 (!coherent && (size & arm_dcache_align_mask)) ||
+		 (!coherent && (paddr & arm_dcache_align_mask)))
 		 && (dmat->filter == NULL
 		  || (*dmat->filter)(dmat->filterarg, paddr) != 0))
 			retval = 1;
@@ -426,8 +426,7 @@ bus_dma_tag_create(bus_dma_tag_t parent,
 	 || newtag->alignment > 1)
 		newtag->flags |= BUS_DMA_COULD_BOUNCE;
 
-	if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
-	    (flags & BUS_DMA_ALLOCNOW) != 0) {
+	if ((flags & BUS_DMA_ALLOCNOW) != 0) {
 		struct bounce_zone *bz;
 
 		/* Must bounce */
@@ -503,6 +502,41 @@ out:
 	return (error);
 }
 
+static int allocate_bz_and_pages(bus_dma_tag_t dmat, bus_dmamap_t mapp)
+{
+        struct bounce_zone *bz;
+	int maxpages;
+	int error;
+		
+	if (dmat->bounce_zone == NULL)
+		if ((error = alloc_bounce_zone(dmat)) != 0)
+			return (error);
+	bz = dmat->bounce_zone;
+	/* Initialize the new map */
+	STAILQ_INIT(&(mapp->bpages));
+
+	/*
+	 * Attempt to add pages to our pool on a per-instance
+	 * basis up to a sane limit.
+	 */
+	maxpages = MAX_BPAGES;
+	if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0
+	    || (bz->map_count > 0 && bz->total_bpages < maxpages)) {
+		int pages;
+		
+		pages = MAX(atop(dmat->maxsize), 1);
+		pages = MIN(maxpages - bz->total_bpages, pages);
+		pages = MAX(pages, 1);
+		if (alloc_bounce_pages(dmat, pages) < pages)
+			return (ENOMEM);
+		
+		if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0)
+			dmat->flags |= BUS_DMA_MIN_ALLOC_COMP;
+	}
+	bz->map_count++;
+	return (0);
+}
+
 /*
  * Allocate a handle for mapping from kva/uva/physical
  * address space into bus device space.
@@ -511,9 +545,7 @@ int
 bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
 {
 	int mapsize;
-	int error;
-
-	error = 0;
+	int error = 0;
 
 	mapsize = sizeof(**mapp) + (sizeof(struct sync_list) * dmat->nsegments);
 	*mapp = (bus_dmamap_t)malloc(mapsize, M_DEVBUF, M_NOWAIT | M_ZERO);
@@ -540,52 +572,12 @@ bus_dmamap_create(bus_dma_tag_t dmat, in
 	 * exclusion region, a data alignment that is stricter than 1, and/or
 	 * an active address boundary.
 	 */
-	if (dmat->flags & BUS_DMA_COULD_BOUNCE) {
-
-		/* Must bounce */
-		struct bounce_zone *bz;
-		int maxpages;
-
-		if (dmat->bounce_zone == NULL) {
-			if ((error = alloc_bounce_zone(dmat)) != 0) {
-				free(*mapp, M_DEVBUF);
-				*mapp = NULL;
-				return (error);
-			}
-		}
-		bz = dmat->bounce_zone;
-
-		/* Initialize the new map */
-		STAILQ_INIT(&((*mapp)->bpages));
-
-		/*
-		 * Attempt to add pages to our pool on a per-instance
-		 * basis up to a sane limit.
-		 */
-		maxpages = MAX_BPAGES;
-		if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0
-		 || (bz->map_count > 0 && bz->total_bpages < maxpages)) {
-			int pages;
-
-			pages = MAX(atop(dmat->maxsize), 1);
-			pages = MIN(maxpages - bz->total_bpages, pages);
-			pages = MAX(pages, 1);
-			if (alloc_bounce_pages(dmat, pages) < pages)
-				error = ENOMEM;
-
-			if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) {
-				if (error == 0)
-					dmat->flags |= BUS_DMA_MIN_ALLOC_COMP;
-			} else {
-				error = 0;
-			}
-		}
-		bz->map_count++;
+	error = allocate_bz_and_pages(dmat, *mapp);
+	if (error != 0) {
+		free(*mapp, M_DEVBUF);
+		*mapp = NULL;
+		return (error);
 	}
-	if (error == 0)
-		dmat->map_count++;
-	CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
-	    __func__, dmat, dmat->flags, error);
 	return (error);
 }
 
@@ -624,6 +616,7 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, voi
 	vm_memattr_t memattr;
 	int mflags;
 	int mapsize;
+	int error;
 
 	if (flags & BUS_DMA_NOWAIT)
 		mflags = M_NOWAIT;
@@ -641,6 +634,13 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, voi
 	}
 
 	(*mapp)->sync_count = 0;
+	/* We may need bounce pages, even for allocated memory */
+	error = allocate_bz_and_pages(dmat, *mapp);
+	if (error != 0) {
+		free(*mapp, M_DEVBUF);
+		*mapp = NULL;
+		return (error);
+	}
 
 	if (dmat->segments == NULL) {
 		dmat->segments = (bus_dma_segment_t *)malloc(
@@ -666,10 +666,6 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, voi
 		ba = standard_allocator;
 		(*mapp)->flags = 0;
 	}
-#ifdef notyet
-	/* All buffers we allocate are cache-aligned. */
-	map->flags |= DMAMAP_CACHE_ALIGNED;
-#endif
 
 	/*
 	 * Try to find a bufzone in the allocator that holds a cache of buffers
@@ -770,7 +766,8 @@ _bus_dmamap_count_phys(bus_dma_tag_t dma
 		curaddr = buf;
 		while (buflen != 0) {
 			sgsize = MIN(buflen, dmat->maxsegsz);
-			if (run_filter(dmat, curaddr) != 0) {
+			if (run_filter(dmat, curaddr, sgsize, 
+			    map->flags & DMAMAP_COHERENT) != 0) {
 				sgsize = MIN(sgsize, PAGE_SIZE);
 				map->pagesneeded++;
 			}
@@ -806,7 +803,10 @@ _bus_dmamap_count_pages(bus_dma_tag_t dm
 				paddr = pmap_kextract(vaddr);
 			else
 				paddr = pmap_extract(map->pmap, vaddr);
-			if (run_filter(dmat, paddr) != 0) {
+			if (run_filter(dmat, paddr,
+			    min(vendaddr - vaddr, 
+			    (PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK))),
+			    map->flags & DMAMAP_COHERENT) != 0) {
 				map->pagesneeded++;
 			}
 			vaddr += (PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK));
@@ -922,7 +922,8 @@ _bus_dmamap_load_phys(bus_dma_tag_t dmat
 	if (segs == NULL)
 		segs = dmat->segments;
 
-	if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
+	if (((map->flags & DMAMAP_COHERENT) == 0) ||
+	    (dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
 		_bus_dmamap_count_phys(dmat, map, buf, buflen, flags);
 		if (map->pagesneeded != 0) {
 			error = _bus_dmamap_reserve_pages(dmat, map, flags);
@@ -934,8 +935,10 @@ _bus_dmamap_load_phys(bus_dma_tag_t dmat
 	while (buflen > 0) {
 		curaddr = buf;
 		sgsize = MIN(buflen, dmat->maxsegsz);
-		if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
-		    map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
+		if ((((map->flags & DMAMAP_COHERENT) == 0) ||
+		    ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0)) &&
+		    map->pagesneeded != 0 && run_filter(dmat, curaddr,
+		    sgsize, map->flags & DMAMAP_COHERENT)) {
 			sgsize = MIN(sgsize, PAGE_SIZE);
 			curaddr = add_bounce_page(dmat, map, 0, curaddr,
 						  sgsize);
@@ -982,7 +985,8 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dm
 
 	map->pmap = pmap;
 
-	if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
+	if (!(map->flags & DMAMAP_COHERENT) ||
+	    (dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) {
 		_bus_dmamap_count_pages(dmat, map, buf, buflen, flags);
 		if (map->pagesneeded != 0) {
 			error = _bus_dmamap_reserve_pages(dmat, map, flags);
@@ -1012,8 +1016,10 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dm
 		if (buflen < sgsize)
 			sgsize = buflen;
 
-		if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
-		    map->pagesneeded != 0 && run_filter(dmat, curaddr)) {
+		if ((((map->flags & DMAMAP_COHERENT) == 0) ||
+		    ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0)) &&
+		    map->pagesneeded != 0 && run_filter(dmat, curaddr,
+		    sgsize, map->flags & DMAMAP_COHERENT)) {
 			curaddr = add_bounce_page(dmat, map, vaddr, curaddr,
 						  sgsize);
 		} else {
@@ -1156,12 +1162,6 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus
 {
 	struct bounce_page *bpage;
 	struct sync_list *sl, *end;
-	bus_size_t len, unalign;
-	vm_offset_t buf, ebuf;
-#ifdef FIX_DMAP_BUS_DMASYNC_POSTREAD
-	vm_offset_t bbuf;
-	char _tmp_cl[arm_dcache_align], _tmp_clend[arm_dcache_align];
-#endif
 	/*
 	 * If the buffer was from user space, it is possible that this is not
 	 * the same vm map, especially on a POST operation.  It's not clear that
@@ -1252,45 +1252,9 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus
 
 		case BUS_DMASYNC_PREREAD:
 			while (sl != end) {
-					/* write back the unaligned portions */
-				vm_paddr_t physaddr = sl->busaddr, ephysaddr;
-				buf = sl->vaddr;
-				len = sl->datacount;
-				ebuf = buf + len;	/* end of buffer */
-				ephysaddr = physaddr + len;
-				unalign = buf & arm_dcache_align_mask;
-				if (unalign) {
-						/* wbinv leading fragment */
-					buf &= ~arm_dcache_align_mask;
-					physaddr &= ~arm_dcache_align_mask;
-					cpu_dcache_wbinv_range(buf,
-							arm_dcache_align);
-					l2cache_wbinv_range(buf, physaddr,
-					    arm_dcache_align);
-					buf += arm_dcache_align;
-					physaddr += arm_dcache_align;
-					/* number byte in buffer wbinv */
-					unalign = arm_dcache_align - unalign;
-					if (len > unalign)
-						len -= unalign;
-					else
-						len = 0;
-				}
-				unalign = ebuf & arm_dcache_align_mask;
-				if (ebuf > buf && unalign) {
-						/* wbinv trailing fragment */
-					len -= unalign;
-					ebuf -= unalign;
-					ephysaddr -= unalign;
-					cpu_dcache_wbinv_range(ebuf,
-							arm_dcache_align);
-					l2cache_wbinv_range(ebuf, ephysaddr,
-					    arm_dcache_align);
-				}
-				if (ebuf > buf) {
-					cpu_dcache_inv_range(buf, len);
-					l2cache_inv_range(buf, physaddr, len);
-				}
+				cpu_dcache_inv_range(sl->vaddr, sl->datacount);
+				l2cache_inv_range(sl->vaddr, sl->busaddr, 
+				    sl->datacount);
 				sl++;
 			}
 			break;
@@ -1304,60 +1268,6 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus
 			}
 			break;
 
-#ifdef FIX_DMAP_BUS_DMASYNC_POSTREAD
-		case BUS_DMASYNC_POSTREAD:
-			while (sl != end) {
-					/* write back the unaligned portions */
-				vm_paddr_t physaddr;
-				register_t s = 0;
-
-				buf = sl->vaddr;
-				len = sl->datacount;
-				physaddr = sl->busaddr;
-				bbuf = buf & ~arm_dcache_align_mask;
-				ebuf = buf + len;
-				physaddr = physaddr & ~arm_dcache_align_mask;
-
-
-				if ((buf & arm_dcache_align_mask) ||
-				    (ebuf & arm_dcache_align_mask)) {
-					s = intr_disable();
-					unalign = buf & arm_dcache_align_mask;
-					if (unalign) {
-						memcpy(_tmp_cl, (void *)bbuf, unalign);
-						len += unalign; /* inv entire cache line */
-					}
-
-					unalign = ebuf & arm_dcache_align_mask;
-					if (unalign) {
-						unalign = arm_dcache_align - unalign;
-						memcpy(_tmp_clend, (void *)ebuf, unalign);
-						len += unalign; /* inv entire cache line */
-					}
-				}
-
-				/* inv are cache length aligned */
-				cpu_dcache_inv_range(bbuf, len);
-				l2cache_inv_range(bbuf, physaddr, len);
-
-				if ((buf & arm_dcache_align_mask) ||
-				    (ebuf & arm_dcache_align_mask)) {
-					unalign = (vm_offset_t)buf & arm_dcache_align_mask;
-					if (unalign)
-						memcpy((void *)bbuf, _tmp_cl, unalign);
-
-					unalign = ebuf & arm_dcache_align_mask;
-					if (unalign)
-						memcpy((void *)ebuf, _tmp_clend,
-						    arm_dcache_align - unalign);
-
-					intr_restore(s);
-				}
-				sl++;
-			}
-				break;
-#endif /* FIX_DMAP_BUS_DMASYNC_POSTREAD */
-
 		default:
 			break;
 		}



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201308071544.r77FiwMK072982>