Date: Tue, 7 Dec 2010 16:30:53 +0000 (UTC) From: Poul-Henning Kamp <phk@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r216257 - head/usr.sbin/fifolog/lib Message-ID: <201012071630.oB7GUr92073186@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: phk Date: Tue Dec 7 16:30:52 2010 New Revision: 216257 URL: http://svn.freebsd.org/changeset/base/216257 Log: Use a "push" strategy to get data through libz, rather than a "pull" strategy. Modified: head/usr.sbin/fifolog/lib/fifolog_write.h head/usr.sbin/fifolog/lib/fifolog_write_poll.c Modified: head/usr.sbin/fifolog/lib/fifolog_write.h ============================================================================== --- head/usr.sbin/fifolog/lib/fifolog_write.h Tue Dec 7 15:24:08 2010 (r216256) +++ head/usr.sbin/fifolog/lib/fifolog_write.h Tue Dec 7 16:30:52 2010 (r216257) @@ -44,8 +44,6 @@ struct fifolog_writer { unsigned syncrate; unsigned compression; - unsigned writes_since_sync; - int cleanup; intmax_t cnt[FIFOLOG_NPOINT]; @@ -55,9 +53,11 @@ struct fifolog_writer { int flag; time_t last; + u_int obufsize; + u_char *obuf; + u_int ibufsize; u_char *ibuf; - u_char *iptr; time_t starttime; time_t lastwrite; Modified: head/usr.sbin/fifolog/lib/fifolog_write_poll.c ============================================================================== --- head/usr.sbin/fifolog/lib/fifolog_write_poll.c Tue Dec 7 15:24:08 2010 (r216256) +++ head/usr.sbin/fifolog/lib/fifolog_write_poll.c Tue Dec 7 16:30:52 2010 (r216257) @@ -33,6 +33,9 @@ #include <unistd.h> #include <time.h> #include <sys/endian.h> +#if 0 +#include <sys/uio.h> +#endif #include <zlib.h> @@ -65,9 +68,8 @@ fifolog_write_assert(const struct fifolo { CHECK_OBJ_NOTNULL(f, FIFOLOG_WRITER_MAGIC); - assert(f->iptr == f->ff->zs->next_in + f->ff->zs->avail_in); assert(f->ff->zs->next_out + f->ff->zs->avail_out == \ - f->ff->recbuf + f->ff->recsize); + f->obuf + f->obufsize); } struct fifolog_writer * @@ -75,8 +77,8 @@ fifolog_write_new(void) { struct fifolog_writer *f; - ALLOC(&f, sizeof *f); - f->magic = FIFOLOG_WRITER_MAGIC; + ALLOC_OBJ(f, FIFOLOG_WRITER_MAGIC); + assert(f != NULL); return (f); } @@ -94,36 +96,11 @@ fifolog_write_close(struct fifolog_write CHECK_OBJ_NOTNULL(f, FIFOLOG_WRITER_MAGIC); fifolog_int_close(&f->ff); free(f->ff); - if (f->ibuf != NULL) - free(f->ibuf); + if (f->obuf != NULL) + free(f->obuf); free(f); } -static void -fifo_prepobuf(struct fifolog_writer *f, time_t now, int flag) -{ - - memset(f->ff->recbuf, 0, f->ff->recsize); - f->ff->zs->next_out = f->ff->recbuf + 5; - f->ff->zs->avail_out = f->ff->recsize - 5; - if (f->recno == 0 && f->seq == 0) { - srandomdev(); - do { - f->seq = random(); - } while (f->seq == 0); - } - be32enc(f->ff->recbuf, f->seq++); - f->ff->recbuf[4] = f->flag; - f->flag = 0; - if (flag) { - f->ff->recbuf[4] |= FIFOLOG_FLG_SYNC; - be32enc(f->ff->recbuf + 5, (u_int)now); - f->ff->zs->next_out += 4; - f->ff->zs->avail_out -= 4; - } - fifolog_write_assert(f); -} - const char * fifolog_write_open(struct fifolog_writer *f, const char *fn, unsigned writerate, unsigned syncrate, int compression) { @@ -164,144 +141,154 @@ fifolog_write_open(struct fifolog_writer f->seq++; } - f->ibufsize = 32768; - ALLOC(&f->ibuf, f->ibufsize); - f->iptr = f->ibuf; - f->ff->zs->next_in = f->iptr; + f->obufsize = f->ff->recsize; + ALLOC(&f->obuf, f->obufsize); + i = deflateInit(f->ff->zs, (int)f->compression); assert(i == Z_OK); f->flag |= FIFOLOG_FLG_RESTART; + f->flag |= FIFOLOG_FLG_SYNC; + f->ff->zs->next_out = f->obuf + 9; + f->ff->zs->avail_out = f->obufsize - 9; time(&now); - fifo_prepobuf(f, now, 1); f->starttime = now; + f->lastsync = now; + f->lastwrite = now; fifolog_write_assert(f); return (NULL); } -static void -fifo_writerec(struct fifolog_writer *f) +static int +fifolog_write_output(struct fifolog_writer *f, int fl, time_t now) { - int i; - time_t t; + long h, l = f->ff->zs->next_out - f->obuf; + int i, w; - fifolog_write_assert(f); - f->writes_since_sync++; + h = 4; /* seq */ + be32enc(f->obuf, f->seq); + f->obuf[h] = f->flag; + h += 1; /* flag */ + if (f->flag & FIFOLOG_FLG_SYNC) { + be32enc(f->obuf + h, now); + h += 4; /* timestamp */ + } - assert(f->recno < f->ff->logsize); - f->cnt[FIFOLOG_PT_BYTES_POST] += f->ff->recsize - f->ff->zs->avail_out; - if (f->ff->zs->avail_out == 0) { - /* nothing */ - } else if (f->ff->zs->avail_out <= 255) { - f->ff->recbuf[f->ff->recsize - 1] = - (u_char)f->ff->zs->avail_out; - f->ff->recbuf[4] |= FIFOLOG_FLG_1BYTE; - } else { - be32enc(f->ff->recbuf + f->ff->recsize - 4, - f->ff->zs->avail_out); - f->ff->recbuf[4] |= FIFOLOG_FLG_4BYTE; + assert(l <= (long)f->ff->recsize); + assert(l >= h); + if (l == h) + return (0); + + + if (h + l < (long)f->ff->recsize && fl == Z_NO_FLUSH) + return (0); + + w = f->ff->recsize - l; + if (w > 255) { + be32enc(f->obuf + f->ff->recsize - 4, w); + f->obuf[4] |= FIFOLOG_FLG_4BYTE; + } else if (w > 0) { + f->obuf[f->ff->recsize - 1] = w; + f->obuf[4] |= FIFOLOG_FLG_1BYTE; } - i = pwrite(f->ff->fd, f->ff->recbuf, f->ff->recsize, - (f->recno + 1) * f->ff->recsize); - assert (i == (int)f->ff->recsize); - if (++f->recno == f->ff->logsize) - f->recno = 0; + + f->cnt[FIFOLOG_PT_BYTES_POST] += w; + +#ifdef DBG +fprintf(stderr, "W: fl=%d h=%ld l=%ld w=%d recno=%jd fx %02x\n", + fl, h, l, w, f->recno, f->obuf[4]); +#endif + + i = pwrite(f->ff->fd, f->obuf, f->ff->recsize, + (f->recno + 1) * f->ff->recsize); + assert(i == (int)f->ff->recsize); + f->cnt[FIFOLOG_PT_WRITES]++; - time(&t); - f->cnt[FIFOLOG_PT_RUNTIME] = t - f->starttime; /*lint !e776 */ - fifolog_write_assert(f); + + f->lastwrite = now; + f->seq++; + f->recno++; +#ifdef DBG +if (f->flag) +fprintf(stderr, "SYNC- %d\n", __LINE__); +#endif + f->flag = 0; + + memset(f->obuf, 0, f->obufsize); + f->ff->zs->next_out = f->obuf + 5; + f->ff->zs->avail_out = f->obufsize - 5; + return (1); } -int -fifolog_write_poll(struct fifolog_writer *f, time_t now) +static void +fifolog_write_gzip(struct fifolog_writer *f, const void *p, int len, time_t now, int fin) { - int i, fl, bo, bf; + int i, fl; - if (now == 0) - time(&now); + f->cnt[FIFOLOG_PT_BYTES_PRE] += len; - fifolog_write_assert(f); - if (f->cleanup || now >= (int)(f->lastsync + f->syncrate)) { - /* - * We always check the sync timer, otherwise a flood of data - * would not get any sync records at all - */ + if (fin == 0) + fl = Z_NO_FLUSH; + else if (f->cleanup || now >= (int)(f->lastsync + f->syncrate)) { f->cleanup = 0; fl = Z_FINISH; - f->lastsync = now; - f->lastwrite = now; f->cnt[FIFOLOG_PT_SYNC]++; - } else if (f->ff->zs->avail_in == 0 && - now >= (int)(f->lastwrite + f->writerate)) { - /* - * We only check for writerate timeouts when the input - * buffer is empty. It would be silly to force a write if - * pending input could cause it to happen on its own. - */ + } else if (now >= (int)(f->lastwrite + f->writerate)) { fl = Z_SYNC_FLUSH; - f->lastwrite = now; f->cnt[FIFOLOG_PT_FLUSH]++; - } else if (f->ff->zs->avail_in == 0) - return (0); /* nothing to do */ + } else if (p == NULL) + return; else fl = Z_NO_FLUSH; - for (;;) { - assert(f->ff->zs->avail_out > 0); - - bf = f->ff->zs->avail_out; + f->ff->zs->avail_in = len; + f->ff->zs->next_in = (void*)(uintptr_t)p; +#ifdef DBG +if (fl != Z_NO_FLUSH) +fprintf(stderr, "Z len %3d fin %d now %ld fl %d ai %u ao %u\n", + len, fin, now, fl, + f->ff->zs->avail_in, + f->ff->zs->avail_out); +#endif + while (1) { i = deflate(f->ff->zs, fl); - assert (i == Z_OK || i == Z_BUF_ERROR || i == Z_STREAM_END); - bo = f->ff->zs->avail_out; +#ifdef DBG +if (i || f->ff->zs->avail_in) +fprintf(stderr, "fl = %d, i = %d ai = %u ao = %u fx=%02x\n", fl, i, + f->ff->zs->avail_in, + f->ff->zs->avail_out, f->flag); +#endif - /* If we have output space and not in a hurry.. */ - if (bo > 0 && fl == Z_NO_FLUSH) - break; - - /* Write output buffer, if anything in it */ - if (bo != bf) - fifo_writerec(f); - - /* If the buffer were full, we need to check again */ - if (bo == 0) { - fifo_prepobuf(f, now, 0); - continue; - } + assert(i == Z_OK || i == Z_BUF_ERROR || i == Z_STREAM_END); + assert(f->ff->zs->avail_in == 0); - if (fl == Z_FINISH) { - /* Make next record a SYNC record */ - fifo_prepobuf(f, now, 1); - /* And reset the zlib engine */ - i = deflateReset(f->ff->zs); - assert(i == Z_OK); - f->writes_since_sync = 0; - } else { - fifo_prepobuf(f, now, 0); - } - break; + if (!fifolog_write_output(f, fl, now)) + break; } - - if (f->ff->zs->avail_in == 0) { - /* Reset input buffer when empty */ - f->iptr = f->ibuf; - f->ff->zs->next_in = f->iptr; + assert(f->ff->zs->avail_in == 0); + if (fl == Z_FINISH) { + f->flag |= FIFOLOG_FLG_SYNC; + f->ff->zs->next_out = f->obuf + 9; + f->ff->zs->avail_out = f->obufsize - 9; + f->lastsync = now; +#ifdef DBG +fprintf(stderr, "SYNC %d\n", __LINE__); +#endif + assert(Z_OK == deflateReset(f->ff->zs)); } - - fifolog_write_assert(f); - return (1); } -static void -fifolog_acct(struct fifolog_writer *f, unsigned bytes) +int +fifolog_write_poll(struct fifolog_writer *f, time_t now) { - - f->ff->zs->avail_in += bytes; - f->iptr += bytes; - f->cnt[FIFOLOG_PT_BYTES_PRE] += bytes; + if (now == 0) + time(&now); + fifolog_write_gzip(f, NULL, 0, now, 1); + return (0); } /* @@ -312,8 +299,8 @@ fifolog_acct(struct fifolog_writer *f, u int fifolog_write_bytes(struct fifolog_writer *f, uint32_t id, time_t now, const void *ptr, unsigned len) { - u_int l; const unsigned char *p; + uint8_t buf[4]; fifolog_write_assert(f); assert(!(id & (FIFOLOG_TIMESTAMP|FIFOLOG_LENGTH))); @@ -322,46 +309,45 @@ fifolog_write_bytes(struct fifolog_write p = ptr; if (len == 0) { len = strlen(ptr) + 1; - l = 4 + len; /* id */ } else { assert(len <= 255); id |= FIFOLOG_LENGTH; - l = 5 + len; /* id + len */ } - l += 4; /* A timestamp may be necessary */ - /* Now do timestamp, if needed */ if (now == 0) time(&now); - assert(l < f->ibufsize); - - /* Return if there is not enough space */ - if (f->iptr + l > f->ibuf + f->ibufsize) - return (0); - if (now != f->last) { id |= FIFOLOG_TIMESTAMP; f->last = now; } - /* Emit instance+flag and length */ - be32enc(f->iptr, id); - fifolog_acct(f, 4); + /* Emit instance+flag */ + be32enc(buf, id); + fifolog_write_gzip(f, buf, 4, now, 0); if (id & FIFOLOG_TIMESTAMP) { - be32enc(f->iptr, (uint32_t)f->last); - fifolog_acct(f, 4); + be32enc(buf, (uint32_t)f->last); + fifolog_write_gzip(f, buf, 4, now, 0); } if (id & FIFOLOG_LENGTH) { - f->iptr[0] = (u_char)len; - fifolog_acct(f, 1); + buf[0] = (u_char)len; + fifolog_write_gzip(f, buf, 1, now, 0); } assert (len > 0); - memcpy(f->iptr, p, len); - fifolog_acct(f, len); +#if 1 + if (len > f->ibufsize) { + free(f->ibuf); + f->ibufsize = len; + ALLOC(&f->ibuf, f->ibufsize); + } + memcpy(f->ibuf, p, len); + fifolog_write_gzip(f, f->ibuf, len, now, 1); +#else + fifolog_write_gzip(f, p, len, now, 1); +#endif fifolog_write_assert(f); return (1); } @@ -384,7 +370,6 @@ fifolog_write_bytes_poll(struct fifolog_ if (len == 0) { while (!fifolog_write_bytes(f, id, now, ptr, len)) { - (void)fifolog_write_poll(f, now); (void)usleep(10000); } } else { @@ -394,7 +379,6 @@ fifolog_write_bytes_poll(struct fifolog_ if (l > 255) l = 255; while (!fifolog_write_bytes(f, id, now, p, l)) { - (void)fifolog_write_poll(f, now); (void)usleep(10000); } }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201012071630.oB7GUr92073186>