Date: Sun, 13 Dec 2020 05:34:15 +0000 (UTC) From: Alan Somers <asomers@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r368605 - in projects/aio_writev/sys: kern sys Message-ID: <202012130534.0BD5YFM7091936@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: asomers Date: Sun Dec 13 05:34:14 2020 New Revision: 368605 URL: https://svnweb.freebsd.org/changeset/base/368605 Log: Finish the bio path for aio_writev. Still needs a few more tests Modified: projects/aio_writev/sys/kern/vfs_aio.c projects/aio_writev/sys/sys/aio.h Modified: projects/aio_writev/sys/kern/vfs_aio.c ============================================================================== --- projects/aio_writev/sys/kern/vfs_aio.c Sun Dec 13 03:58:43 2020 (r368604) +++ projects/aio_writev/sys/kern/vfs_aio.c Sun Dec 13 05:34:14 2020 (r368605) @@ -1223,21 +1223,25 @@ aio_qbio(struct proc *p, struct kaiocb *job) { struct aiocb *cb; struct file *fp; - struct bio *bp; struct buf *pbuf; struct vnode *vp; struct cdevsw *csw; struct cdev *dev; struct kaioinfo *ki; - int error, ref, poff; + struct uio *auiop = NULL; + off_t offset; + size_t nbytes = 0; + int bio_cmd, error, i, opcode, ref, poff, iovcnt; + bool vectored; vm_prot_t prot; cb = &job->uaiocb; fp = job->fd_file; + opcode = cb->aio_lio_opcode; - // TODO: handle LIO_WRITEV - if (!(cb->aio_lio_opcode == LIO_WRITE || - cb->aio_lio_opcode == LIO_READ)) + if (!(opcode == LIO_WRITE || + opcode == LIO_WRITEV || + opcode == LIO_READ)) return (-1); if (fp == NULL || fp->f_type != DTYPE_VNODE) return (-1); @@ -1247,105 +1251,141 @@ aio_qbio(struct proc *p, struct kaiocb *job) return (-1); if (vp->v_bufobj.bo_bsize == 0) return (-1); - if (cb->aio_nbytes % vp->v_bufobj.bo_bsize) - return (-1); + bio_cmd = opcode == LIO_WRITE || opcode == LIO_WRITEV ? BIO_WRITE : + BIO_READ; + vectored = opcode == LIO_WRITEV; + if (vectored) { + iovcnt = cb->aio_iovcnt; + if (iovcnt > max_buf_aio) + return (-1); + error = copyinuio(cb->aio_iov, cb->aio_iovcnt, &auiop); + if (error) + return (error); + nbytes = auiop->uio_resid; + } else { + nbytes = cb->aio_nbytes; + iovcnt = 1; + } + offset = cb->aio_offset; + + if (nbytes % vp->v_bufobj.bo_bsize) { + error = -1; + goto free_uio; + } + ref = 0; csw = devvn_refthread(vp, &dev, &ref); - if (csw == NULL) - return (ENXIO); + if (csw == NULL) { + error = ENXIO; + goto free_uio; + } if ((csw->d_flags & D_DISK) == 0) { error = -1; goto unref; } - if (cb->aio_nbytes > dev->si_iosize_max) { + if (nbytes > dev->si_iosize_max) { error = -1; goto unref; } ki = p->p_aioinfo; - poff = (vm_offset_t)cb->aio_buf & PAGE_MASK; - if ((dev->si_flags & SI_UNMAPPED) && unmapped_buf_allowed) { - if (cb->aio_nbytes > maxphys) { + job->error = 0; + atomic_store_int(&job->nbio, iovcnt); + for (i = 0; i < iovcnt; i++) { + struct vm_page** pages; + struct bio *bp; + void *buf; + size_t nbytes; + int npages; + + bp = g_alloc_bio(); + + if (vectored) { + buf = auiop->uio_iov[i].iov_base; + nbytes = auiop->uio_iov[i].iov_len; + } else { + buf = (void *)(uintptr_t)cb->aio_buf; + nbytes = cb->aio_nbytes; + } + if (nbytes > maxphys) { error = -1; goto unref; } + poff = (vm_offset_t)buf & PAGE_MASK; + if ((dev->si_flags & SI_UNMAPPED) && unmapped_buf_allowed) { + pbuf = NULL; + pages = malloc(sizeof(vm_page_t) * (atop(round_page( + nbytes)) + 1), M_TEMP, M_WAITOK | M_ZERO); + } else { + if (ki->kaio_buffer_count + iovcnt > max_buf_aio) { + error = EAGAIN; + goto unref; + } - pbuf = NULL; - job->pages = malloc(sizeof(vm_page_t) * (atop(round_page( - cb->aio_nbytes)) + 1), M_TEMP, M_WAITOK | M_ZERO); - } else { - if (cb->aio_nbytes > maxphys) { - error = -1; - goto unref; + pbuf = uma_zalloc(pbuf_zone, M_WAITOK); + BUF_KERNPROC(pbuf); + AIO_LOCK(ki); + ki->kaio_buffer_count++; + AIO_UNLOCK(ki); + pages = pbuf->b_pages; } - if (ki->kaio_buffer_count >= max_buf_aio) { - error = EAGAIN; + + bp->bio_length = nbytes; + bp->bio_bcount = nbytes; + bp->bio_done = aio_biowakeup; + bp->bio_offset = offset; + bp->bio_cmd = bio_cmd; + bp->bio_dev = dev; + bp->bio_caller1 = (void *)job; + bp->bio_caller2 = (void *)pbuf; + + prot = VM_PROT_READ; + if (cb->aio_lio_opcode == LIO_READ) + prot |= VM_PROT_WRITE; /* Less backwards than it looks */ + npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, + (vm_offset_t)buf, bp->bio_length, prot, pages, + atop(maxphys) + 1); + if (npages < 0) { + if (pbuf != NULL) { + AIO_LOCK(ki); + ki->kaio_buffer_count--; + AIO_UNLOCK(ki); + uma_zfree(pbuf_zone, pbuf); + } else { + free(pages, M_TEMP); + } + g_destroy_bio(bp); + error = EFAULT; goto unref; } + if (pbuf != NULL) { + pmap_qenter((vm_offset_t)pbuf->b_data, pages, npages); + bp->bio_data = pbuf->b_data + poff; + atomic_add_int(&num_buf_aio, 1); + } else { + bp->bio_ma = pages; + bp->bio_ma_n = npages; + bp->bio_ma_offset = poff; + bp->bio_data = unmapped_buf; + bp->bio_flags |= BIO_UNMAPPED; + atomic_add_int(&num_unmapped_aio, 1); + } - job->pbuf = pbuf = uma_zalloc(pbuf_zone, M_WAITOK); - BUF_KERNPROC(pbuf); - AIO_LOCK(ki); - ki->kaio_buffer_count++; - AIO_UNLOCK(ki); - job->pages = pbuf->b_pages; - } - /* For LIO_WRITEV, create a parent bio with multiple child bios */ - job->bp = bp = g_alloc_bio(); + /* Perform transfer. */ + csw->d_strategy(bp); - bp->bio_length = cb->aio_nbytes; - bp->bio_bcount = cb->aio_nbytes; - bp->bio_done = aio_biowakeup; - bp->bio_offset = cb->aio_offset; - bp->bio_cmd = cb->aio_lio_opcode == LIO_WRITE ? BIO_WRITE : BIO_READ; - bp->bio_dev = dev; - bp->bio_caller1 = (void *)job; - - prot = VM_PROT_READ; - if (cb->aio_lio_opcode == LIO_READ) - prot |= VM_PROT_WRITE; /* Less backwards than it looks */ - job->npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, - (vm_offset_t)cb->aio_buf, bp->bio_length, prot, job->pages, - atop(maxphys) + 1); - if (job->npages < 0) { - error = EFAULT; - goto doerror; + offset += nbytes; } - if (pbuf != NULL) { - pmap_qenter((vm_offset_t)pbuf->b_data, - job->pages, job->npages); - bp->bio_data = pbuf->b_data + poff; - atomic_add_int(&num_buf_aio, 1); - } else { - bp->bio_ma = job->pages; - bp->bio_ma_n = job->npages; - bp->bio_ma_offset = poff; - bp->bio_data = unmapped_buf; - bp->bio_flags |= BIO_UNMAPPED; - atomic_add_int(&num_unmapped_aio, 1); - } - - /* Perform transfer. */ - csw->d_strategy(bp); dev_relthread(dev, ref); + free(auiop, M_IOV); return (0); -doerror: - if (pbuf != NULL) { - AIO_LOCK(ki); - ki->kaio_buffer_count--; - AIO_UNLOCK(ki); - uma_zfree(pbuf_zone, pbuf); - job->pbuf = NULL; - } else { - free(job->pages, M_TEMP); - } - g_destroy_bio(bp); - job->bp = NULL; unref: dev_relthread(dev, ref); +free_uio: + free(auiop, M_IOV); return (error); } @@ -2367,44 +2407,48 @@ aio_biowakeup(struct bio *bp) { struct kaiocb *job = (struct kaiocb *)bp->bio_caller1; struct kaioinfo *ki; + struct buf *pbuf = (struct buf*)bp->bio_caller2;; size_t nbytes; int error, nblks; /* Release mapping into kernel space. */ - if (job->pbuf != NULL) { - pmap_qremove((vm_offset_t)job->pbuf->b_data, job->npages); - vm_page_unhold_pages(job->pages, job->npages); - uma_zfree(pbuf_zone, job->pbuf); - job->pbuf = NULL; + if (pbuf != NULL) { + pmap_qremove((vm_offset_t)pbuf->b_data, bp->bio_ma_n); + vm_page_unhold_pages(bp->bio_ma, bp->bio_ma_n); + uma_zfree(pbuf_zone, pbuf); atomic_subtract_int(&num_buf_aio, 1); ki = job->userproc->p_aioinfo; AIO_LOCK(ki); ki->kaio_buffer_count--; AIO_UNLOCK(ki); } else { - vm_page_unhold_pages(job->pages, job->npages); - free(job->pages, M_TEMP); + vm_page_unhold_pages(bp->bio_ma, bp->bio_ma_n); + free(bp->bio_ma, M_TEMP); atomic_subtract_int(&num_unmapped_aio, 1); } - bp = job->bp; - job->bp = NULL; - nbytes = job->uaiocb.aio_nbytes - bp->bio_resid; + nbytes = bp->bio_bcount - bp->bio_resid; + atomic_add_acq_long(&job->nbytes, nbytes); + nblks = btodb(nbytes); error = 0; + // If multiple bios experienced an error, the job will reflect the error + // of whichever failed bio completed last. if (bp->bio_flags & BIO_ERROR) - error = bp->bio_error; - nblks = btodb(nbytes); + atomic_set_int(&job->error, bp->bio_error); if (job->uaiocb.aio_lio_opcode == LIO_WRITE) - job->outblock += nblks; + atomic_add_int(&job->outblock, nblks); else - job->inblock += nblks; + atomic_add_int(&job->inblock, nblks); + atomic_subtract_int(&job->nbio, 1); - if (error) - aio_complete(job, -1, error); - else - aio_complete(job, nbytes, 0); - g_destroy_bio(bp); + + if (atomic_load_int(&job->nbio) == 0) { + if (atomic_load_int(&job->error)) + aio_complete(job, -1, job->error); + else + aio_complete(job, atomic_load_int(&job->nbytes), 0); + } } /* syscall - wait for the next completion of an aio request */ Modified: projects/aio_writev/sys/sys/aio.h ============================================================================== --- projects/aio_writev/sys/sys/aio.h Sun Dec 13 03:58:43 2020 (r368604) +++ projects/aio_writev/sys/sys/aio.h Sun Dec 13 05:34:14 2020 (r368605) @@ -146,10 +146,9 @@ struct kaiocb { aio_handle_fn_t *handle_fn; /* (c) backend handle function */ union { /* Backend-specific data fields */ struct { /* BIO backend */ - struct bio *bp; /* (*) BIO pointer */ - struct buf *pbuf; /* (*) buffer pointer */ - int npages; /* (*) number of pages */ - struct vm_page **pages; /* (*) */ + int nbio; /* Number of remaining bios */ + int error; /* Worst error of all bios */ + long nbytes; /* Bytes completed so far */ }; struct { /* fsync() requests */ int pending; /* (a) number of pending I/O */
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202012130534.0BD5YFM7091936>