Skip site navigation (1)Skip section navigation (2)
Date:      Sun, 21 Aug 2011 13:02:49 GMT
From:      Kirk Russell <kirk@ba23.org>
To:        freebsd-gnats-submit@FreeBSD.org
Subject:   i386/159971: panic with soft updates journaling during load testing
Message-ID:  <201108211302.p7LD2nGD092322@red.freebsd.org>
Resent-Message-ID: <201108211310.p7LDA4Wx044410@freefall.freebsd.org>

next in thread | raw e-mail | index | archive | help

>Number:         159971
>Category:       i386
>Synopsis:       panic with soft updates journaling during load testing
>Confidential:   no
>Severity:       non-critical
>Priority:       low
>Responsible:    freebsd-i386
>State:          open
>Quarter:        
>Keywords:       
>Date-Required:
>Class:          sw-bug
>Submitter-Id:   current-users
>Arrival-Date:   Sun Aug 21 13:10:04 UTC 2011
>Closed-Date:
>Last-Modified:
>Originator:     Kirk Russell
>Release:        FreeBSD 9.0-BETA1
>Organization:
bstg
>Environment:
FreeBSD kleenex 9.0-BETA1 FreeBSD 9.0-BETA1 #0: Fri Aug 12 21:31:10 IST 2011     root@kleenex:/usr/obj/usr/src/sys/GENERIC  i386
>Description:
I have been testing a scratch filesystem, with soft updates journaling enabled.
I have been seeing one of these two panics:
    panic: ino 0xc5d0f600(0x3C8209) 14147, 7047 != 14098
and
    panic: Bad link elm 0xc4d7cd00 prev->next != elm
If I disable soft updates journaling, I do not see these panics.

Here is the source to the load test:
http://www.ba23.org/bstgbugs/bstg0003.c

Here is the output from /var/crash/core.txt:
http://www.ba23.org/bstgbugs/bstg0003.core.txt.gz

panic: ino 0xc5835000(0x3C8209) 1847043, 2320643 != 1420808

(kgdb) #0  doadump (textdump=0) at pcpu.h:244
#1  0xc04e4683 in db_dump (dummy=-1063023718, dummy2=0, dummy3=-1, 
    dummy4=0xed6de6f8 "") at /usr/src/sys/ddb/db_command.c:537
#2  0xc04e3da1 in db_command (last_cmdp=0xc104fc7c, cmd_table=0x0, dopager=1)
    at /usr/src/sys/ddb/db_command.c:448
#3  0xc04e3efa in db_command_loop () at /usr/src/sys/ddb/db_command.c:501
#4  0xc04e5eed in db_trap (type=3, code=0) at /usr/src/sys/ddb/db_main.c:229
#5  0xc0a38d63 in kdb_trap (type=3, code=0, tf=0xed6de8a8)
    at /usr/src/sys/kern/subr_kdb.c:539
#6  0xc0d347bb in trap (frame=0xed6de8a8) at /usr/src/sys/i386/i386/trap.c:719
#7  0xc0d1d69c in calltrap () at /usr/src/sys/i386/i386/exception.s:168
#8  0xc0a38b9a in kdb_enter (why=0xc0eefcd5 "panic", msg=0xc0eefcd5 "panic")
    at cpufunc.h:71
#9  0xc0a04274 in panic (fmt=0xc0f24586 "ino %p(0x%X) %d, %d != %d")
    at /usr/src/sys/kern/kern_shutdown.c:587
#10 0xc0c35964 in softdep_disk_io_initiation (bp=0xc499e89c)
    at /usr/src/sys/ufs/ffs/ffs_softdep.c:9818
#11 0xc0c3d9af in ffs_geom_strategy (bo=0xc5493e7c, bp=0xc499e89c)
    at buf.h:405
#12 0xc0a85e79 in bufwrite (bp=0xc499e89c) at buf.h:398
#13 0xc0c3cfc0 in ffs_bufwrite (bp=0xc499e89c)
    at /usr/src/sys/ufs/ffs/ffs_vfsops.c:2074
#14 0xc0c1b23c in ffs_update (vp=0xc5820220, waitfor=2) at buf.h:386
#15 0xc0c44113 in ffs_syncvnode (vp=0xc5820220, waitfor=2)
    at /usr/src/sys/ufs/ffs/ffs_vnops.c:304
#16 0xc0c3e11f in ffs_sync (mp=0xc5230a20, waitfor=2)
    at /usr/src/sys/ufs/ffs/ffs_vfsops.c:1498
#17 0xc0aa56f0 in sync (td=0xc55082e0, uap=0xed6decec)
    at /usr/src/sys/kern/vfs_syscalls.c:149
#18 0xc0a47663 in syscallenter (td=0xc55082e0, sa=0xed6dece4)
    at /usr/src/sys/kern/subr_trap.c:344
#19 0xc0d34064 in syscall (frame=0xed6ded28)
    at /usr/src/sys/i386/i386/trap.c:1082
#20 0xc0d1d701 in Xint0x80_syscall ()
    at /usr/src/sys/i386/i386/exception.s:266
#21 0x00000033 in ?? ()

>How-To-Repeat:
- as root -- setup the filesystem with soft updates journaling
# newfs -j /dev/ada0p4
/dev/ada0p4: 56320.0MB (115343360 sectors) block size 32768, fragment size 4096
        using 77 cylinder groups of 740.00MB, 23680 blks, 47360 inodes.
        with soft updates
super-block backups (for fsck -b #) at:
 192, 1515712, 3031232, 4546752, 6062272, 7577792, 9093312, 10608832, 12124352,
 13639872, 15155392, 16670912, 18186432, 19701952, 21217472, 22732992,
 24248512, 25764032, 27279552, 28795072, 30310592, 31826112, 33341632,
 34857152, 36372672, 37888192, 39403712, 40919232, 42434752, 43950272,
 45465792, 46981312, 48496832, 50012352, 51527872, 53043392, 54558912,
 56074432, 57589952, 59105472, 60620992, 62136512, 63652032, 65167552,
 66683072, 68198592, 69714112, 71229632, 72745152, 74260672, 75776192,
 77291712, 78807232, 80322752, 81838272, 83353792, 84869312, 86384832,
 87900352, 89415872, 90931392, 92446912, 93962432, 95477952, 96993472,
 98508992, 100024512, 101540032, 103055552, 104571072, 106086592, 107602112,
 109117632, 110633152, 112148672, 113664192, 115179712
Using inode 4 in cg 0 for 33554432 byte journal
newfs: soft updates journaling set
# mount /dev/ada0p4 /mnt
# chmod -R a+wrx /mnt
chmod: /mnt/.sujournal: Operation not permitted

- as a regular user, compile and run the load test
$ cat bstg0003.c
/*
 * Copyright 2011 Kirk J. Russell
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <unistd.h>
#include <assert.h>
#include <fcntl.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/uio.h>
#include <sys/wait.h>

static char * bstg_pathstore[] = {
    "/mnt/111/z",
    "/mnt/111/aaaa",
    "/mnt/111/bbbbb",
    "/mnt/111/ccccc",
    "/mnt/111/d",
    "/mnt/111/e",
    "/mnt/111/ffffff.fff.f",
    "/mnt/111/gggggggggggg",
    "/mnt/111/hhhh",
    "/mnt/111/iiiii.ii",
    "/mnt/111/jjjj.jj.jjjjjjjj",
    "/mnt/111/kkkk.kkkkkkkk",
    "/mnt/111/lllll",
    "/mnt/222/z",
    "/mnt/222/aaaa",
    "/mnt/222/bbbbb",
    "/mnt/222/ccccc",
    "/mnt/222/d",
    "/mnt/222/e",
    "/mnt/222/ffffff.fff.f",
    "/mnt/222/gggggggggggg",
    "/mnt/222/hhhh",
    "/mnt/222/iiiii.ii",
    "/mnt/222/jjjj.jj.jjjjjjjj",
    "/mnt/222/kkkk.kkkkkkkk",
    "/mnt/222/lllll",
    "/mnt/333/z",
    "/mnt/333/aaaa",
    "/mnt/333/bbbbb",
    "/mnt/333/ccccc",
    "/mnt/333/d",
    "/mnt/333/e",
    "/mnt/333/ffffff.fff.f",
    "/mnt/333/gggggggggggg",
    "/mnt/333/hhhh",
    "/mnt/333/iiiii.ii",
    "/mnt/333/jjjj.jj.jjjjjjjj",
    "/mnt/333/kkkk.kkkkkkkk",
    "/mnt/333/lllll",
    "/mnt/444/z",
    "/mnt/444/aaaa",
    "/mnt/444/bbbbb",
    "/mnt/444/ccccc",
    "/mnt/444/d",
    "/mnt/444/e",
    "/mnt/444/ffffff.fff.f",
    "/mnt/444/gggggggggggg",
    "/mnt/444/hhhh",
    "/mnt/444/iiiii.ii",
    "/mnt/444/jjjj.jj.jjjjjjjj",
    "/mnt/444/kkkk.kkkkkkkk",
    "/mnt/444/lllll",
    "/mnt/555/z",
    "/mnt/555/aaaa",
    "/mnt/555/bbbbb",
    "/mnt/555/ccccc",
    "/mnt/555/d",
    "/mnt/555/e",
    "/mnt/555/ffffff.fff.f",
    "/mnt/555/gggggggggggg",
    "/mnt/555/hhhh",
    "/mnt/555/iiiii.ii",
    "/mnt/555/jjjj.jj.jjjjjjjj",
    "/mnt/555/kkkk.kkkkkkkk",
    "/mnt/555/lllll",
    "/mnt/666/z",
    "/mnt/666/aaaa",
    "/mnt/666/bbbbb",
    "/mnt/666/ccccc",
    "/mnt/666/d",
    "/mnt/666/e",
    "/mnt/666/ffffff.fff.f",
    "/mnt/666/gggggggggggg",
    "/mnt/666/hhhh",
    "/mnt/666/iiiii.ii",
    "/mnt/666/jjjj.jj.jjjjjjjj",
    "/mnt/666/kkkk.kkkkkkkk",
    "/mnt/666/lllll",
    "/mnt/777/z",
    "/mnt/777/aaaa",
    "/mnt/777/bbbbb",
    "/mnt/777/ccccc",
    "/mnt/777/d",
    "/mnt/777/e",
    "/mnt/777/ffffff.fff.f",
    "/mnt/777/gggggggggggg",
    "/mnt/777/hhhh",
    "/mnt/777/iiiii.ii",
    "/mnt/777/jjjj.jj.jjjjjjjj",
    "/mnt/777/kkkk.kkkkkkkk",
    "/mnt/777/lllll",
    "/mnt/888/z",
    "/mnt/888/aaaa",
    "/mnt/888/bbbbb",
    "/mnt/888/ccccc",
    "/mnt/888/d",
    "/mnt/888/e",
    "/mnt/888/ffffff.fff.f",
    "/mnt/888/gggggggggggg",
    "/mnt/888/hhhh",
    "/mnt/888/iiiii.ii",
    "/mnt/888/jjjj.jj.jjjjjjjj",
    "/mnt/888/kkkk.kkkkkkkk",
    "/mnt/888/lllll",
    "/mnt/999/z",
    "/mnt/999/aaaa",
    "/mnt/999/bbbbb",
    "/mnt/999/ccccc",
    "/mnt/999/d",
    "/mnt/999/e",
    "/mnt/999/ffffff.fff.f",
    "/mnt/999/gggggggggggg",
    "/mnt/999/hhhh",
    "/mnt/999/iiiii.ii",
    "/mnt/999/jjjj.jj.jjjjjjjj",
    "/mnt/999/kkkk.kkkkkkkk",
    "/mnt/999/lllll",
    "/mnt/aaa/z",
    "/mnt/aaa/aaaa",
    "/mnt/aaa/bbbbb",
    "/mnt/aaa/ccccc",
    "/mnt/aaa/d",
    "/mnt/aaa/e",
    "/mnt/aaa/ffffff.fff.f",
    "/mnt/aaa/gggggggggggg",
    "/mnt/aaa/hhhh",
    "/mnt/aaa/iiiii.ii",
    "/mnt/aaa/jjjj.jj.jjjjjjjj",
    "/mnt/aaa/kkkk.kkkkkkkk",
    "/mnt/aaa/lllll",
    "/mnt/bbb/z",
    "/mnt/bbb/aaaa",
    "/mnt/bbb/bbbbb",
    "/mnt/bbb/ccccc",
    "/mnt/bbb/d",
    "/mnt/bbb/e",
    "/mnt/bbb/ffffff.fff.f",
    "/mnt/bbb/gggggggggggg",
    "/mnt/bbb/hhhh",
    "/mnt/bbb/iiiii.ii",
    "/mnt/bbb/jjjj.jj.jjjjjjjj",
    "/mnt/bbb/kkkk.kkkkkkkk",
    "/mnt/bbb/lllll",
    "/mnt/ccc/z",
    "/mnt/ccc/aaaa",
    "/mnt/ccc/bbbbb",
    "/mnt/ccc/ccccc",
    "/mnt/ccc/d",
    "/mnt/ccc/e",
    "/mnt/ccc/ffffff.fff.f",
    "/mnt/ccc/gggggggggggg",
    "/mnt/ccc/hhhh",
    "/mnt/ccc/iiiii.ii",
    "/mnt/ccc/jjjj.jj.jjjjjjjj",
    "/mnt/ccc/kkkk.kkkkkkkk",
    "/mnt/ccc/lllll",
    "/mnt/ddd/z",
    "/mnt/ddd/aaaa",
    "/mnt/ddd/bbbbb",
    "/mnt/ddd/ccccc",
    "/mnt/ddd/d",
    "/mnt/ddd/e",
    "/mnt/ddd/ffffff.fff.f",
    "/mnt/ddd/gggggggggggg",
    "/mnt/ddd/hhhh",
    "/mnt/ddd/iiiii.ii",
    "/mnt/ddd/jjjj.jj.jjjjjjjj",
    "/mnt/ddd/kkkk.kkkkkkkk",
    "/mnt/ddd/lllll",
    "/mnt/eee/z",
    "/mnt/eee/aaaa",
    "/mnt/eee/bbbbb",
    "/mnt/eee/ccccc",
    "/mnt/eee/d",
    "/mnt/eee/e",
    "/mnt/eee/ffffff.fff.f",
    "/mnt/eee/gggggggggggg",
    "/mnt/eee/hhhh",
    "/mnt/eee/iiiii.ii",
    "/mnt/eee/jjjj.jj.jjjjjjjj",
    "/mnt/eee/kkkk.kkkkkkkk",
    "/mnt/eee/lllll",
    "/mnt/fff/z",
    "/mnt/fff/aaaa",
    "/mnt/fff/bbbbb",
    "/mnt/fff/ccccc",
    "/mnt/fff/d",
    "/mnt/fff/e",
    "/mnt/fff/ffffff.fff.f",
    "/mnt/fff/gggggggggggg",
    "/mnt/fff/hhhh",
    "/mnt/fff/iiiii.ii",
    "/mnt/fff/jjjj.jj.jjjjjjjj",
    "/mnt/fff/kkkk.kkkkkkkk",
    "/mnt/fff/lllll"
};

char *
bstg_pathstore_get()
{
    return bstg_pathstore[rand() %
        ((sizeof(bstg_pathstore)/sizeof(bstg_pathstore[0])))];
}

void
dogcore()
{
    pid_t sleepchild, gcorechild;
    extern char **environ;

    /* create a child for the gcore target */
    if ((sleepchild = fork()) == 0) {
        sleep(30);
        _exit(1);
    } else if (sleepchild > 0) {
        char *token[] = { NULL, NULL, NULL, NULL, NULL };
        char buf[64];
        int status;

        /* use the first process as the target */
        snprintf(buf, sizeof(buf), "%d", sleepchild);
        token[0] = "gcore";
        token[1] = "-c";
        token[2] = bstg_pathstore_get();
        token[3] = buf;
        assert(token[4] == NULL);

        if ((gcorechild = fork()) > 0) {
            waitpid(gcorechild, &status, 0);
        } else if (gcorechild == 0) {
            execve("/usr/bin/gcore", token, environ);
            _exit(1);
        }

        kill(sleepchild, SIGKILL);
        waitpid(sleepchild, &status, 0);
    }
}

void
dowrite()
{
    struct iovec data[] = {
        { "12", 2 },
        { NULL, 0 },
        { "12345678", 8},
    };
    static int fd = -1;

    if (fd == -1) {
        /* keep existing file open during life of this process */
        fd = open(bstg_pathstore_get(), O_RDWR|O_NONBLOCK|O_NOCTTY);
    }

    data[1].iov_base = bstg_pathstore_get();
    data[1].iov_len = strlen((char*)data[1].iov_base);
    ftruncate(fd, 0);
    pwritev(fd, data, 3, 0);
}

void
dounlink()
{
    unlink(bstg_pathstore_get());
}

void
dolink()
{
    link(bstg_pathstore_get(), bstg_pathstore_get());
}

void
domkdir()
{
    char **pdir;
    static char * bstg_dirs[] = {
        "/mnt/111", "/mnt/222", "/mnt/333", "/mnt/444",
        "/mnt/555", "/mnt/666", "/mnt/777", "/mnt/888",
        "/mnt/999", "/mnt/aaa", "/mnt/bbb", "/mnt/ccc",
        "/mnt/ddd", "/mnt/eee", "/mnt/fff", NULL
    };

    for (pdir = bstg_dirs; *pdir; pdir++) {
        mkdir(*pdir, 0777);
    }
}

void
dosync()
{
    sync();
}

int
main()
{
    unsigned x;
    int status;
    void (*funcs[])() = {
        dogcore,
        dowrite,
        dounlink,
        dolink,
        dowrite,
        dounlink,
        dolink,
        dowrite,
        dosync,
        dowrite,
        dounlink,
        dolink,
        dowrite,
        dounlink,
        dolink,
        dowrite,
    };

    /* we only can domkdir() once at startup */
    domkdir();

    /* create 128 children that loop forever running 4 operations */
    dosync();
    for (x = 0; x < 128; x++) {
        if (fork() == 0) {
            /* give child a new seed for the pathname selection */
            srand(x);

            for (;;) {
                /* each child will start looping at different function */
                (*funcs[x++ % 16]) ();
            }
            /* we never expect this code to run */
            _exit(1);
        }
    }

    /* block forever for all our children */
    while(wait(&status) > 0);
    return 0;
}
$ cc -Wall -O2 bstg0003.c
$ ./a.out
[... takes a few minutes to cause the panic ...]
>Fix:


>Release-Note:
>Audit-Trail:
>Unformatted:



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201108211302.p7LD2nGD092322>