Date: Fri, 22 Oct 2004 12:52:51 -0400 (EDT) From: Mikhail Teterin <mi@aldan.algebra.com> To: FreeBSD-gnats-submit@FreeBSD.org Subject: kern/73010: vnode_pager_getpages: unexpected missing page Message-ID: <200410221652.i9MGqpQb000752@250-217.customer.cloud9.net> Resent-Message-ID: <200410221700.i9MH0cxV062972@freefall.freebsd.org>
next in thread | raw e-mail | index | archive | help
>Number: 73010
>Category: kern
>Synopsis: vnode_pager_getpages: unexpected missing page
>Confidential: no
>Severity: critical
>Priority: medium
>Responsible: freebsd-bugs
>State: open
>Quarter:
>Keywords:
>Date-Required:
>Class: sw-bug
>Submitter-Id: current-users
>Arrival-Date: Fri Oct 22 17:00:38 GMT 2004
>Closed-Date:
>Last-Modified:
>Originator: Mikhail Teterin
>Release: FreeBSD 6.0-CURRENT i386
>Organization:
Virtual Estates, Inc.
>Environment:
System: FreeBSD mi 6.0-CURRENT FreeBSD 6.0-CURRENT #1: Wed Oct 20 12:08:24 EDT 2004 mteterin@mi:/meow/obj/misha/src/sys/Gigabyte i386
>Description:
I'm seeing this panic for the second time in a recent current.
The first time was on amd64 (reported to current@ last week).
This is on i386. The `firstaddr' reported by panic was -1 in
both cases.
My program makes heavy use of mmap -- mapping huge chunks of
input and output to pass them to -lbz2 (or -lz) in big portions.
If the output file is not big enough, it is ftruncate()-ed up.
Whether the program is somehow wrong or not (it works fine for
smaller files 4Mb), it should not cause a panic, right?
>How-To-Repeat:
Compile the following program (LDADD=-lbz2) and run it on
a large file -- something well above 4Gb.
To reproduce on amd64, you may need to increase MAX_INPUT_MAP
further -- not sure.
#include <assert.h>
#include <err.h>
#include <errno.h>
#include <fcntl.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <sys/stat.h>
#include <sysexits.h>
#include <bzlib.h>
#include <zlib.h>
#define MAX_INPUT_MAP 0x1F000000
#define MAX_OUTPUT_MAP MAX_INPUT_MAP
union {
z_stream gz;
bz_stream bz;
} stream;
enum FResponse {
OK, ERROR, MORE
};
typedef void (Init)(int level);
typedef enum FResponse (Filter)(char *dest, size_t destLen,
const char *src, size_t srcLen, size_t *eaten, size_t *produced);
typedef off_t (Finish)(void);
static Init initbz, initgz;
static Filter compressbz2, decompressbz2, compressz, decompressz;
static Finish finishbz, finishgz;
static void
initbz(int level)
{
stream.bz.bzalloc = NULL;
stream.bz.bzfree = NULL;
if (BZ2_bzCompressInit(&stream.bz, level, 0, 0) != BZ_OK) {
fputs("BZ2_bzCompressInit failed\n", stderr);
exit(EX_SOFTWARE);
}
}
static void
initgz(int level)
{
stream.gz.zalloc = NULL;
stream.gz.zfree = NULL;
if (deflateInit(&stream.gz, level) != Z_OK) {
fputs("deflateInit failed\n", stderr);
exit(EX_SOFTWARE);
}
}
static void
usage(const char *prog, int code)
{
fprintf(stderr, "Usage:\n\t%s [-g] [-d] [-[1-9]] input output\n"
"Where the options mean:\n"
"\t-g\tuse zlib instead of the default bzlib (not yet)\n"
"\t-d\tdecompress input into output instead of compressing (n/y)\n"
"\t1-9\tspecifies the desired compression level\n",
prog);
exit(code);
}
struct mzf {
char *start;
off_t size, offset;
size_t mapped;
int fd;
} input = {0}, output = {0};
static void
mapfile(const char *name, int mode, struct mzf *result)
{
struct stat sb;
if (result->mapped == 0) {
if (mode != O_RDONLY)
mode |= O_CREAT;
result->fd = open(name, mode, S_IRUSR|S_IWUSR);
if (result->fd == -1) {
perror(name);
exit(EX_NOINPUT);
}
} else {
if (munmap(result->start, result->mapped))
perror("warning: munmap");
result->mapped = 0;
}
/* fstat again -- the size may have changed */
if (fstat(result->fd, &sb)) {
perror("fstat");
exit(EX_OSERR);
}
if (mode == O_RDONLY) {
off_t mapped = sb.st_size - result->offset;
if (result->size && sb.st_size != result->size) {
if (sb.st_size < result->size) {
fprintf(stderr, "the input size shrunk from "
"%jd to %jd. I'm not prepared for this\n",
result->size, sb.st_size);
exit(EX_NOINPUT);
}
fprintf(stderr, "the input size grew from %jd to "
"%jd. Ok...\n", result->size, sb.st_size);
}
if (mapped < MAX_INPUT_MAP)
result->mapped = mapped;
else
result->mapped = MAX_INPUT_MAP;
if (mapped == 0) {
/* We reached the end of the file happily */
result->start = NULL;
return;
}
result->size = sb.st_size;
} else {
result->mapped = MAX_OUTPUT_MAP;
result->size = MAX_OUTPUT_MAP + result->offset;
if (ftruncate(result->fd, result->size)) {
perror("Adjusting output size");
exit(EX_OSERR);
}
}
fprintf(stderr, "mmap-ing %lu bytes of %s (%jd) starting at %jd\n",
(unsigned long)result->mapped, name, sb.st_size, result->offset);
result->start = mmap(NULL, result->mapped,
mode == O_RDONLY ? PROT_READ : PROT_WRITE,
MAP_NOCORE|MAP_SHARED|MAP_NOSYNC, result->fd, result->offset);
if (result->start == MAP_FAILED) {
perror("mmap");
exit(EX_OSERR);
}
}
static enum FResponse
compressbz2(char *dest, size_t destLen, const char *src, size_t srcLen,
size_t *eaten, size_t *produced)
{
int code;
stream.bz.next_in = (char *)src; /* XXX bzlib does not use const */
stream.bz.avail_in = srcLen;
stream.bz.next_out = dest;
stream.bz.avail_out = destLen;
code = BZ2_bzCompress(&stream.bz, srcLen ? BZ_RUN : BZ_FINISH);
*eaten = stream.bz.next_in - src;
*produced = stream.bz.next_out - dest;
fprintf(stderr, "(eaten: %lu of %lu, produced %lu)%s",
(unsigned long)*eaten, (unsigned long)srcLen,
(unsigned long)*produced,
srcLen ? "\n" : " (finishing ...");
switch (code) {
case BZ_RUN_OK:
return MORE;
case BZ_FINISH_OK:
fprintf(stderr, " will need another run)\n");
return MORE;
case BZ_STREAM_END:
fprintf(stderr, " done)\n");
return OK;
default:
fprintf(stderr, "BZ2_bzCompress: unexpected result: %d\n",
code);
return ERROR;
}
}
static off_t
finishbz() {
int code;
off_t size;
code = BZ2_bzCompressEnd(&stream.bz);
if (code != BZ_OK) {
fprintf(stderr, "BZ2_bzCompressEnd failed: %d\n", code);
exit(EX_SOFTWARE);
}
size = stream.bz.total_out_hi32;
size <<= 32;
size += stream.bz.total_out_lo32;
fprintf(stderr, "Total compressed seems to be %jd bytes\n", size);
return size;
}
static enum FResponse
decompressbz2(char *dest, size_t destLen, const char *src, size_t srcLen,
size_t *eaten, size_t *produced)
{
exit(EX_SOFTWARE);
}
static off_t
finishgz()
{
exit(EX_SOFTWARE);
}
static enum FResponse
compressz(char *dest, size_t destLen, const char *src, size_t srcLen,
size_t *eaten, size_t *produced)
{
exit(EX_SOFTWARE);
}
static enum FResponse
decompressz(char *dest, size_t destLen, const char *src, size_t srcLen,
size_t *eaten, size_t *produced)
{
exit(EX_SOFTWARE);
}
static void
cleanup(void)
{
struct rusage ru;
if (input.start && input.start != MAP_FAILED) {
fprintf(stderr, "Unmapping input (%p, %llu)\n",
input.start, (unsigned long long)input.size);
munmap(input.start, input.size);
}
if (output.start && output.start != MAP_FAILED) {
fprintf(stderr, "Unmapping output (%p, %llu)\n",
output.start, (unsigned long long)output.size);
munmap(output.start, output.size);
}
if (getrusage(RUSAGE_SELF, &ru))
perror("getrusage");
else
fprintf(stderr, "execution time = %ld.%lds utime, "
"%ld.%lds stime; memory used: %ldKb, pfaults: %ld\n",
ru.ru_utime.tv_sec, ru.ru_utime.tv_usec/1000,
ru.ru_stime.tv_sec, ru.ru_stime.tv_usec/1000,
ru.ru_maxrss*getpagesize()/1024, ru.ru_majflt);
}
int
main(int argc, char *argv[])
{
Filter *update = compressbz2;
Init *init = initbz;
Finish *finish = finishbz;
int level = 9, opt, watch = 0;
off_t truesize;
size_t eaten = 0, produced = 0;
while ((opt = getopt(argc, argv, "whdg123456789")) != -1) {
switch (opt) {
case 'd':
if (update == compressbz2)
update = decompressbz2;
else if (update == compressz)
update = decompressz;
else
usage(argv[0], EX_USAGE);
break;
case 'g':
if (update != compressbz2)
usage(argv[0], EX_USAGE);
update = compressz;
init = initgz;
finish = finishgz;
break;
case '1':case '2':case '3':case '4':
case '5':case '6':case '7':case '8':case '9':
level = opt - '0';
break;
case 'w':
watch = 1;
break;
case 'h':
usage(argv[0], EX_OK);
default:
usage(argv[0], EX_USAGE);
}
}
if (argc - optind != 2)
usage(argv[0], EX_USAGE);
argv += optind;
argc -= optind;
atexit(cleanup);
init(level);
truesize = 0;
for (;;) {
size_t in, out;
if (eaten == input.mapped) {
input.offset += eaten;
mapfile(argv[0], O_RDONLY, &input);
eaten = 0;
}
if (produced == output.mapped) {
output.offset += produced;
mapfile(argv[1], O_WRONLY, &output);
produced = 0;
}
fprintf(stderr, "filtering: %p-%jd, %p-%jd\n",
output.start + produced, (off_t)output.mapped - produced,
input.start + eaten, (off_t)input.mapped - eaten);
switch (update(output.start + produced,
output.mapped - produced, input.start + eaten,
input.mapped - eaten, &in, &out)) {
case OK:
fprintf(stderr, "Got Ok. Exiting loop\n");
break;
case MORE:
eaten += in;
produced += out;
continue;
case ERROR:
unlink(argv[1]);
exit(EX_SOFTWARE);
}
break;
}
truesize = finish();
fprintf(stderr, "Truncating output to exactly %ju bytes\n",
truesize);
ftruncate(output.fd, truesize);
fsync(output.fd);
close(output.fd);
return EX_OK;
}
>Fix:
>Release-Note:
>Audit-Trail:
>Unformatted:
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200410221652.i9MGqpQb000752>