Skip site navigation (1)Skip section navigation (2)
Date:      Fri, 26 Mar 2010 20:10:00 +0000
From:      Tom Judge <tom@tomjudge.com>
To:        Kostik Belousov <kostikbel@gmail.com>
Cc:        freebsd-hackers@freebsd.org
Subject:   Re: Panic in vm_map_stack
Message-ID:  <4BAD1498.5040402@tomjudge.com>
In-Reply-To: <20100326195659.GU2415@deviant.kiev.zoral.com.ua>
References:  <4BACF92E.60600@tomjudge.com> <20100326195659.GU2415@deviant.kiev.zoral.com.ua>

next in thread | previous in thread | raw e-mail | index | archive | help
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1

Interesting this is in some custom code to calculate and track memory
usage in jails.

#0  sched_switch (td=0xffffff0004325370, newtd=0xffffffff804aeed8,
flags=Variable "flags" is not available.
)
    at /usr/src/sys/kern/sched_ule.c:1944
#1  0x0000000000000000 in ?? ()
#2  0xffffffff804960a1 in mi_switch (flags=70407024, newtd=0x1)
    at /usr/src/sys/kern/kern_synch.c:440
#3  0xffffffff804c24a0 in sleepq_timedwait (wchan=0xffffffff80a09748)
    at /usr/src/sys/kern/subr_sleepqueue.c:615
#4  0xffffffff8049663d in _sleep (ident=0xffffffff80a09748, lock=0x0,
    priority=83314928, wmesg=0xffffffff807c4219 "vmo_de", timo=1)
    at /usr/src/sys/kern/kern_synch.c:222
#5  0xffffffff8064b5d4 in vm_object_deallocate (object=0xffffff07335f8c30)
    at /usr/src/sys/vm/vm_object.c:512
#6  0xffffffff8064440e in vm_map_delete (map=0xffffff007f8191a0,
    start=18446742980708383584, end=140737488355328)
    at /usr/src/sys/vm/vm_map.c:2315
#7  0xffffffff806445df in vm_map_remove (map=0xffffff007f8191a0, start=0,
    end=140737488355328) at /usr/src/sys/vm/vm_map.c:2423
#8  0xffffffff80644bb9 in vmspace_free (vm=0xffffff007f8191a0)
    at /usr/src/sys/vm/vm_map.c:324
#9  0xffffffff80470403 in prison_memory (pr=0xffffff0004fa2000)
    at /usr/src/sys/kern/kern_jail.c:756
#10 0xffffffff804705f8 in jpager_td (arg=Variable "arg" is not available.
) at /usr/src/sys/kern/kern_jail.c:192
#11 0xffffffff8046ad93 in fork_exit (callout=0xffffffff804705a0
<jpager_td>,
    arg=0xffffff0004fa2000, frame=0xffffffffb8986c80)
- ---Type <return> to continue, or q <return> to quit---
    at /usr/src/sys/kern/kern_fork.c:804
#12 0xffffffff806be49e in fork_trampoline ()
    at /usr/src/sys/amd64/amd64/exception.S:455

This is the function, I am guessing that I need to unlock the proc
before calling vmspace_free ?




 673 /* Given credential, return memory usage in bytes. */
 674 void
 675 prison_memory(struct prison *pr)
 676 {
 677     struct proc *p;
 678     struct thread *td;
 679     struct vmspace *vm;
 680     long mem_used = 0;
 681     long full_mem_used = 0;
 682     long proc_res = 0;
 683
 684     /*
 685      * TODO: this is a really bad way of doing the
 686      * search, as we end up going across all processes
 687      * for each jail.  It'd be more efficient to just do
 688      * this once in a period and update the relevant jail.
 689      *
 690      */
 691     sx_slock(&allproc_lock);
 692
 693     FOREACH_PROC_IN_SYSTEM(p) {
 694     int breakout;
 695         proc_res=0;
 696     vm = NULL;
 697         if (PROC_TRYLOCK(p) == 0)
 698         continue;
 699     /*
 700      * If this is a system or protected process, skip it.
 701      */
 702     if ((p->p_flag & P_SYSTEM) || (p->p_pid == 1) ||
 703         (p->p_flag & P_PROTECTED) ||
 704         (p->p_pid < 48)) {
 705         PROC_UNLOCK(p);
 706         continue;
 707     }
 708     /*
 709      * If the process is in a non-running type state,
 710      * don't touch it.  Check all the threads individually.
 711      */
 712     breakout = 0;
 713     FOREACH_THREAD_IN_PROC(p, td) {
 714         thread_lock(td);
 715         if (!TD_ON_RUNQ(td) &&
 716             !TD_IS_RUNNING(td) &&
 717             !TD_IS_SLEEPING(td)) {
 718             thread_unlock(td);
 719             breakout = 1;
 720             break;
 721         }
 722         thread_unlock(td);
 723     }
 724     if (breakout) {
 725         PROC_UNLOCK(p);
 726         continue;
 727     }
 728
 729         if (p->p_state == PRS_NEW ||
 730         p->p_state == PRS_ZOMBIE ||
 731             !jailed(p->p_ucred) ||
 732             (pr != p->p_ucred->cr_prison) ||
 733             !p->p_vmspace) {
 734             PROC_UNLOCK(p);
 735             continue;
 736         }
 737     /*
 738      * get the process size
 739      */
 740     vm = vmspace_acquire_ref(p);
 741     if (vm == NULL) {
 742         PROC_UNLOCK(p);
 743         continue;
 744     }
 745
 746         if (!vm_map_trylock_read(&vm->vm_map)) {
 747         vmspace_free(vm);
 748             PROC_UNLOCK(p);
 749             continue;
 750         }
 751         full_mem_used += vmspace_swap_count(vm);
 752         vm_map_unlock_read(&vm->vm_map);
 753         proc_res = vmspace_resident_count(vm);
 754         full_mem_used += proc_res;
 755         mem_used += proc_res;
 756         vmspace_free(vm);
 757         PROC_UNLOCK(p);
 758     }
 759     sx_sunlock(&allproc_lock);
 760
 761     mem_used *= PAGE_SIZE;
 762     full_mem_used *= PAGE_SIZE;
 763     /* Copy the current memory usage to the prison struct */
 764     mtx_lock(&pr->pr_mtx);
 765     pr->pr_mem_usage = mem_used;
 766     pr->pr_full_mem_usage = full_mem_used;
 767     mtx_unlock(&pr->pr_mtx);
 768 }
 769



Tom


Kostik Belousov wrote:
> On Fri, Mar 26, 2010 at 06:13:02PM +0000, Tom Judge wrote:
>> -----BEGIN PGP SIGNED MESSAGE-----
>> Hash: SHA1
>>
>> Hi,
>>
>> I am seeing the following rare but re occurring panic in the
>> vm_map_stack code.
>>
>> I have the core file and symbols available should more information be
>> required.
>>
>> Tom
>>
>> - ----
>>
>>
>>> $ uname -a
>> FreeBSD XXX 7.1-RELEASE-p4 FreeBSD 7.1-RELEASE-p4 #0 @718:841: Mon Feb
>> 8 20:14:39 UTC 2010     tj@XXX:/usr/obj/usr/src/sys/XXXv8  amd64
>>
>>
>>> $ kgdb /boot/kernel/kernel /var/crash/vmcore.0
>> GNU gdb 6.1.1 [FreeBSD]
>> Copyright 2004 Free Software Foundation, Inc.
>> GDB is free software, covered by the GNU General Public License, and you are
>> welcome to change it and/or distribute copies of it under certain
>> conditions.
>> Type "show copying" to see the conditions.
>> There is absolutely no warranty for GDB.  Type "show warranty" for details.
>> This GDB was configured as "amd64-marcel-freebsd"...
>>
>> Unread portion of the kernel message buffer:
>> Sleeping thread (tid 100119, pid 1554) owns a non-sleepable lock
>> panic: sleeping thread
>> cpuid = 0
>> Uptime: 44d17h36m22s
>> Physical memory: 32746 MB
>> Dumping 1467 MB: 1452 1436 1420 1404 1388 1372 1356 1340 1324 1308 1292
>> 1276 1260 1244 1228 1212 1196 1180 1164 1148 1132 1116 1100 1084 1068
>> 1052 1036 1020 1004 988 972 956 940 924 908 892 876 860 844 828 812 796
>> 780 764 748 732 716 700 684 668 652 636 620 604 588 572 556 540 524 508
>> 492 476 460 444 428 412 396 380 364 348 332 316 300 284 268 252 236 220
>> 204 188 172 156 140 124 108 92 76 60 44 28 12
>>
>> #0  doadump () at pcpu.h:195
>> 195	pcpu.h: No such file or directory.
>> 	in pcpu.h
>> (kgdb) bt
>> #0  doadump () at pcpu.h:195
>> #1  0x0000000000000004 in ?? ()
>> #2  0xffffffff8048e2a9 in boot (howto=260) at
>> /usr/src/sys/kern/kern_shutdown.c:418
>> #3  0xffffffff8048e6b2 in panic (fmt=0x104 <Address 0x104 out of
>> bounds>) at /usr/src/sys/kern/kern_shutdown.c:574
>> #4  0xffffffff804c4a95 in propagate_priority (td=Variable "td" is not
>> available.
>> ) at /usr/src/sys/kern/subr_turnstile.c:222
>> #5  0xffffffff804c58a5 in turnstile_wait (ts=Variable "ts" is not available.
>> ) at /usr/src/sys/kern/subr_turnstile.c:740
>> #6  0xffffffff804816af in _mtx_lock_sleep (m=0xffffff0654567118,
>> tid=18446742976254672896, opts=Variable "opts" is not available.
>> ) at /usr/src/sys/kern/kern_mutex.c:420
>> #7  0xffffffff80646057 in vm_map_stack (map=0xffffff000447cd00,
>> addrbos=140736951484416, max_ssize=536870912, prot=7 '\a', max=7 '\a',
>> cow=4096) at /usr/src/sys/vm/vm_map.c:2722
>> #8  0xffffffff80467115 in exec_new_vmspace (imgp=0xffffffffb92b4a10,
>> sv=0xffffffff809b0a40) at /usr/src/sys/kern/kern_exec.c:971
>> #9  0xffffffff8044e534 in exec_elf64_imgact (imgp=0xffffffffb92b4a10) at
>> /usr/src/sys/kern/imgact_elf.c:680
>> #10 0xffffffff80467532 in kern_execve (td=0xffffff007a978000,
>> args=0xffffffffb92b4b00, mac_p=Variable "mac_p" is not available.
>> ) at /usr/src/sys/kern/kern_exec.c:432
>> #11 0xffffffff80468477 in execve (td=0xffffff007a978000, uap=Variable
>> "uap" is not available.
>> ) at /usr/src/sys/kern/kern_exec.c:201
>> #12 0xffffffff806d8217 in syscall (frame=0xffffffffb92b4c80) at
>> /usr/src/sys/amd64/amd64/trap.c:907
>> #13 0xffffffff806be2db in Xfast_syscall () at
>> /usr/src/sys/amd64/amd64/exception.S:330
>> #14 0x000000080091bcec in ?? ()
>> Previous frame inner to this frame (corrupt stack?)
>> (kgdb)
> 
> Switch to the thread 100119, and get a backtrace for it.
> You can look up the kgdb thread number by kernel tid by
> issuing "threads" command.


- --
TJU13-ARIN
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v2.0.13 (FreeBSD)
Comment: Using GnuPG with Mozilla - http://enigmail.mozdev.org/

iQEcBAEBAgAGBQJLrRSYAAoJEMSwVS7lr0OdAjUH/2pOJl0r7sl79+rDgABt9QLw
IF+geLoM0Htb71xf8FJ6xXZ4rarclrzCx4I1ZJCgmkTE44EKx9929/e0pqDwGbZ+
O+lj+zYs3wrppo8u/GGicJYYyhMblIpVfr/OVvIEtC6otTFwo+scyeXg7VHL2qqG
HaMakCZwR65P1ydbTqz7aSWKay4l1SNq8uoeeaaE6ujVXmUEWYC231IsoirR8L4L
E5teJBlLoilK2rJN0vGvECMXiuvuT4hjraTJXOO99Tzge3BP663We3LI+o4+4Tam
eD6D2cOT2PdrTbvfzLZWCq+EAwolk8MKCZqGLzz5dkFtEIu4Q/JToOu4UEj9GQU=
=//on
-----END PGP SIGNATURE-----



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?4BAD1498.5040402>