Date: Tue, 20 Feb 2018 02:32:22 +0000 (UTC) From: Jeff Roberson <jeff@FreeBSD.org> To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r329618 - in user/jeff/numa/sys: kern sys Message-ID: <201802200232.w1K2WMjT006809@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: jeff Date: Tue Feb 20 02:32:22 2018 New Revision: 329618 URL: https://svnweb.freebsd.org/changeset/base/329618 Log: Catch up with the version in head. Modified: user/jeff/numa/sys/kern/vfs_bio.c user/jeff/numa/sys/sys/buf.h Modified: user/jeff/numa/sys/kern/vfs_bio.c ============================================================================== --- user/jeff/numa/sys/kern/vfs_bio.c Tue Feb 20 02:31:22 2018 (r329617) +++ user/jeff/numa/sys/kern/vfs_bio.c Tue Feb 20 02:32:22 2018 (r329618) @@ -298,7 +298,7 @@ struct bufqueue { struct mtx_padalign bq_lock; TAILQ_HEAD(, buf) bq_queue; uint8_t bq_index; - uint16_t bq_cpu; + uint16_t bq_subqueue; int bq_len; } __aligned(CACHE_LINE_SIZE); @@ -311,8 +311,8 @@ struct bufqueue __exclusive_cache_line bqempty; struct bufqueue __exclusive_cache_line bqdirty; struct bufdomain { - struct bufqueue bd_cpuq[MAXCPU]; - struct bufqueue bd_cleanq; + struct bufqueue bd_subq[MAXCPU + 1]; /* Per-cpu sub queues + global */ + struct bufqueue *bd_cleanq; struct mtx_padalign bd_run_lock; /* Constants */ long bd_maxbufspace; @@ -329,7 +329,7 @@ struct bufdomain { int __aligned(CACHE_LINE_SIZE) bd_freebuffers; } __aligned(CACHE_LINE_SIZE); -#define BD_LOCKPTR(bd) (&(bd)->bd_cleanq.bq_lock) +#define BD_LOCKPTR(bd) (&(bd)->bd_cleanq->bq_lock) #define BD_LOCK(bd) mtx_lock(BD_LOCKPTR((bd))) #define BD_UNLOCK(bd) mtx_unlock(BD_LOCKPTR((bd))) #define BD_ASSERT_LOCKED(bd) mtx_assert(BD_LOCKPTR((bd)), MA_OWNED) @@ -491,7 +491,7 @@ bufspace_daemon_wakeup(struct bufdomain *bd) */ if (atomic_fetchadd_int(&bd->bd_running, 1) == 0) { BD_RUN_LOCK(bd); - bd->bd_running = 1; + atomic_store_int(&bd->bd_running, 1); wakeup(&bd->bd_running); BD_RUN_UNLOCK(bd); } @@ -1041,7 +1041,7 @@ bufinit(void) bp->b_wcred = NOCRED; bp->b_qindex = QUEUE_NONE; bp->b_domain = -1; - bp->b_cpu = -1; + bp->b_subqueue = mp_ncpus; bp->b_xflags = 0; bp->b_data = bp->b_kvabase = unmapped_buf; LIST_INIT(&bp->b_dep); @@ -1330,8 +1330,6 @@ bpmap_qenter(struct buf *bp) static struct bufqueue * bufqueue(struct buf *bp) { - struct bufdomain *bd; - int cpu; switch (bp->b_qindex) { case QUEUE_NONE: @@ -1343,17 +1341,11 @@ bufqueue(struct buf *bp) case QUEUE_DIRTY: return (&bqdirty); case QUEUE_CLEAN: - /* FALLTHROUGH */ - break; + return (&bdclean[bp->b_domain].bd_subq[bp->b_subqueue]); default: - panic("bufqueue(%p): Unhandled type %d\n", bp, bp->b_qindex); + break; } - bd = &bdclean[bp->b_domain]; - /* cpu may be changed by bd_flush(). Read it only once. */ - cpu = bp->b_cpu; - if (cpu > mp_maxid) - return (&bd->bd_cleanq); - return (&bd->bd_cpuq[cpu]); + panic("bufqueue(%p): Unhandled type %d\n", bp, bp->b_qindex); } /* @@ -1415,14 +1407,12 @@ binsfree(struct buf *bp, int qindex) if (qindex == QUEUE_CLEAN) { bd = &bdclean[bp->b_domain]; if (bd->bd_lim != 0) - bq = &bd->bd_cpuq[PCPU_GET(cpuid)]; + bq = &bd->bd_subq[PCPU_GET(cpuid)]; else - bq = &bd->bd_cleanq; + bq = bd->bd_cleanq; } else bq = &bqdirty; bq_insert(bq, bp, true); - - return; } /* @@ -1592,7 +1582,7 @@ buf_recycle(struct bufdomain *bd, bool kva) if (kva) counter_u64_add(bufdefragcnt, 1); nbp = NULL; - bq = &bd->bd_cleanq; + bq = bd->bd_cleanq; BQ_LOCK(bq); KASSERT(BQ_LOCKPTR(bq) == BD_LOCKPTR(bd), ("buf_recycle: Locks don't match")); @@ -1709,14 +1699,14 @@ bremfreef(struct buf *bp) } static void -bq_init(struct bufqueue *bq, int qindex, int cpu, const char *lockname) +bq_init(struct bufqueue *bq, int qindex, int subqueue, const char *lockname) { mtx_init(&bq->bq_lock, lockname, NULL, MTX_DEF); TAILQ_INIT(&bq->bq_queue); bq->bq_len = 0; bq->bq_index = qindex; - bq->bq_cpu = cpu; + bq->bq_subqueue = subqueue; } static void @@ -1726,10 +1716,11 @@ bd_init(struct bufdomain *bd) int i; domain = bd - bdclean; - bq_init(&bd->bd_cleanq, QUEUE_CLEAN, -1, "bufq clean lock"); + bd->bd_cleanq = &bd->bd_subq[mp_ncpus]; + bq_init(bd->bd_cleanq, QUEUE_CLEAN, -1, "bufq clean lock"); for (i = 0; i <= mp_maxid; i++) - bq_init(&bd->bd_cpuq[i], QUEUE_CLEAN, i, - "bufq clean cpu lock"); + bq_init(&bd->bd_subq[i], QUEUE_CLEAN, i, + "bufq clean subqueue lock"); mtx_init(&bd->bd_run_lock, "bufspace daemon run lock", NULL, MTX_DEF); } @@ -1759,7 +1750,7 @@ bq_remove(struct bufqueue *bq, struct buf *bp) TAILQ_REMOVE(&bq->bq_queue, bp, b_freelist); bq->bq_len--; bp->b_qindex = QUEUE_NONE; - bp->b_flags &= ~(B_REMFREE|B_REUSE); + bp->b_flags &= ~(B_REMFREE | B_REUSE); } static void @@ -1768,22 +1759,22 @@ bd_flush(struct bufdomain *bd, struct bufqueue *bq) struct buf *bp; BQ_ASSERT_LOCKED(bq); - if (bq != &bd->bd_cleanq) { + if (bq != bd->bd_cleanq) { BD_LOCK(bd); while ((bp = TAILQ_FIRST(&bq->bq_queue)) != NULL) { TAILQ_REMOVE(&bq->bq_queue, bp, b_freelist); - TAILQ_INSERT_TAIL(&bd->bd_cleanq.bq_queue, bp, + TAILQ_INSERT_TAIL(&bd->bd_cleanq->bq_queue, bp, b_freelist); - bp->b_cpu = -1; + bp->b_subqueue = mp_ncpus; } - bd->bd_cleanq.bq_len += bq->bq_len; + bd->bd_cleanq->bq_len += bq->bq_len; bq->bq_len = 0; } if (bd->bd_wanted) { bd->bd_wanted = 0; wakeup(&bd->bd_wanted); } - if (bq != &bd->bd_cleanq) + if (bq != bd->bd_cleanq) BD_UNLOCK(bd); } @@ -1798,7 +1789,7 @@ bd_flushall(struct bufdomain *bd) return (0); flushed = 0; for (i = 0; i < mp_maxid; i++) { - bq = &bd->bd_cpuq[i]; + bq = &bd->bd_subq[i]; if (bq->bq_len == 0) continue; BQ_LOCK(bq); @@ -1822,7 +1813,7 @@ bq_insert(struct bufqueue *bq, struct buf *bp, bool un if (bp->b_flags & B_AGE) { /* Place this buf directly on the real queue. */ if (bq->bq_index == QUEUE_CLEAN) - bq = &bd->bd_cleanq; + bq = bd->bd_cleanq; BQ_LOCK(bq); TAILQ_INSERT_HEAD(&bq->bq_queue, bp, b_freelist); } else { @@ -1832,7 +1823,7 @@ bq_insert(struct bufqueue *bq, struct buf *bp, bool un bp->b_flags &= ~(B_AGE | B_REUSE); bq->bq_len++; bp->b_qindex = bq->bq_index; - bp->b_cpu = bq->bq_cpu; + bp->b_subqueue = bq->bq_subqueue; /* * Unlock before we notify so that we don't wakeup a waiter that @@ -1845,7 +1836,7 @@ bq_insert(struct bufqueue *bq, struct buf *bp, bool un /* * Flush the per-cpu queue and notify any waiters. */ - if (bd->bd_wanted || (bq != &bd->bd_cleanq && + if (bd->bd_wanted || (bq != bd->bd_cleanq && bq->bq_len >= bd->bd_lim)) bd_flush(bd, bq); } @@ -5207,12 +5198,12 @@ DB_SHOW_COMMAND(bufqueues, bufqueues) db_printf("\tlobufspace\t%ld\n", bd->bd_lobufspace); db_printf("\tbufspacethresh\t%ld\n", bd->bd_bufspacethresh); db_printf("\n"); - db_printf("\tcleanq count\t%d\n", bd->bd_cleanq.bq_len); + db_printf("\tcleanq count\t%d\n", bd->bd_cleanq->bq_len); db_printf("\twakeup\t\t%d\n", bd->bd_wanted); db_printf("\tlim\t\t%d\n", bd->bd_lim); db_printf("\tCPU "); for (j = 0; j < mp_maxid + 1; j++) - db_printf("%d, ", bd->bd_cpuq[j].bq_len); + db_printf("%d, ", bd->bd_subq[j].bq_len); db_printf("\n"); } } Modified: user/jeff/numa/sys/sys/buf.h ============================================================================== --- user/jeff/numa/sys/sys/buf.h Tue Feb 20 02:31:22 2018 (r329617) +++ user/jeff/numa/sys/sys/buf.h Tue Feb 20 02:32:22 2018 (r329618) @@ -114,7 +114,7 @@ struct buf { uint32_t b_vflags; /* (V) BV_* flags */ uint8_t b_qindex; /* (Q) buffer queue index */ uint8_t b_domain; /* (Q) buf domain this resides in */ - uint16_t b_cpu; /* (Q) per-cpu q if any */ + uint16_t b_subqueue; /* (Q) per-cpu q if any */ uint32_t b_flags; /* B_* flags. */ b_xflags_t b_xflags; /* extra flags */ struct lock b_lock; /* Buffer lock */
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201802200232.w1K2WMjT006809>