[CentOS] VM Ware guest shutdown hangs with high load in uninterruptible sleep was RE: Deciphering top's data
Jason Pyeron
jpyeron at pdinc.us
Fri Feb 6 15:58:29 UTC 2009
> -----Original Message-----
> From: centos-bounces at centos.org
> [mailto:centos-bounces at centos.org] On Behalf Of Jason Pyeron
> Sent: Friday, February 06, 2009 10:40
> To: 'CentOS mailing list'
> Subject: Re: [CentOS] Deciphering top's data
>
>
> > -----Original Message-----
> > From: centos-bounces at centos.org
> > [mailto:centos-bounces at centos.org] On Behalf Of Jason Pyeron
> > Sent: Friday, February 06, 2009 10:13
> > To: 'CentOS mailing list'
> > Subject: Re: [CentOS] Deciphering top's data
> >
> >
> >
> > So I issued a reboot.
> >
> > It hung on stoping iptables.
> >
> > I had to power cycle the machine.
> >
> > Machine details:
> > Linux ------------- 2.6.9-55.0.2.ELsmp #1 SMP Tue Jun 26
> > 14:14:47 EDT 2007
> > x86_64 x86_64 x86_64 GNU/Linux
> > CentOS release 4.6 (Final)
>
>
> Here we go again.
>
> UID PID PPID C STIME TTY STAT TIME CMD
> root 4066 1 4 10:01 ? Ds 1:13
> /usr/lib/vmware/bin/vmware-vmx -C /var/lib/vmware/VMs/proxy/RHEL.vmx
>
>
> top - 10:33:59 up 33 min, 2 users, load average: 1.06, 0.67, 0.40
> Tasks: 118 total, 1 running, 117 sleeping, 0 stopped, 0 zombie
> Cpu(s): 0.5% us, 2.0% sy, 0.0% ni, 97.3% id, 0.0% wa,
> 0.0% hi, 0.2% si
> Mem: 1928300k total, 1318688k used, 609612k free,
> 47648k buffers
> Swap: 2031608k total, 0k used, 2031608k free,
> 1046132k cached
>
>
> Ug. This production system.... They are going to have my head.
>
>
>
>
$ lsof -p 4066
COMMAND PID USER FD TYPE DEVICE SIZE NODE NAME
vmware-vm 4066 root cwd DIR 253,0 4096 14483669
/var/lib/vmware/Virtual Machines/proxy
vmware-vm 4066 root rtd DIR 253,0 4096 2 /
vmware-vm 4066 root txt REG 253,0 4385956 4096669
/usr/lib/vmware/bin/vmware-vmx
vmware-vm 4066 root mem REG 253,0 112168 17826955
/lib/ld-2.3.4.so
vmware-vm 4066 root mem REG 253,0 1529120 17826956
/lib/tls/libc-2.3.4.so
vmware-vm 4066 root mem REG 253,0 16732 17826957
/lib/libdl-2.3.4.so
vmware-vm 4066 root mem REG 253,0 63624 3909886
/usr/lib/libz.so.1.2.1.2
vmware-vm 4066 root mem REG 253,0 213772 17826958
/lib/tls/libm-2.3.4.so
vmware-vm 4066 root mem REG 253,0 107800 17828754
/lib/tls/libpthread-2.3.4.so
vmware-vm 4066 root mem REG 253,0 53476 3912514
/usr/X11R6/lib/libXext.so.6.4
vmware-vm 4066 root mem REG 253,0 32720 3912517
/usr/X11R6/lib/libSM.so.6.0
vmware-vm 4066 root mem REG 253,0 86520 3912516
/usr/X11R6/lib/libICE.so.6.3
vmware-vm 4066 root mem REG 253,0 326684 3912518
/usr/X11R6/lib/libXt.so.6.0
vmware-vm 4066 root mem REG 253,0 913020 3905116
/usr/X11R6/lib/libX11.so.6.2
vmware-vm 4066 root mem REG 253,0 29668 3912519
/usr/X11R6/lib/libXrender.so.1.2.2
vmware-vm 4066 root mem REG 253,0 18596 3912515
/usr/X11R6/lib/libXtst.so.6.1
vmware-vm 4066 root mem REG 253,0 38624 17825931
/lib/libgcc_s-3.4.6-20060404.so.1
vmware-vm 4066 root mem CHR 1,1 526 /dev/mem
vmware-vm 4066 root mem REG 253,0 79376 17825849
/lib/libresolv-2.3.4.so
vmware-vm 4066 root mem REG 253,0 22516 17825834
/lib/libnss_dns-2.3.4.so
vmware-vm 4066 root mem REG 253,0 47404 17825837
/lib/libnss_files-2.3.4.so
vmware-vm 4066 root mem REG 253,0 189908 4128894
/usr/lib/vmware/lib/libssl.so.0.9.7/libssl.so.0.9.7
vmware-vm 4066 root mem REG 253,0 1022860 4128781
/usr/lib/vmware/lib/libcrypto.so.0.9.7/libcrypto.so.0.9.7
vmware-vm 4066 root mem REG 253,0 21544 3932925
/usr/lib/gconv/gconv-modules.cache
vmware-vm 4066 root 0u CHR 1,3 2335 /dev/null
vmware-vm 4066 root 1w FIFO 0,7 8833 pipe
vmware-vm 4066 root 2w FIFO 0,7 8833 pipe
vmware-vm 4066 root 3r FIFO 0,7 8848 pipe
vmware-vm 4066 root 4w FIFO 0,7 8848 pipe
vmware-vm 4066 root 5r FIFO 0,7 8849 pipe
vmware-vm 4066 root 6w FIFO 0,7 8849 pipe
vmware-vm 4066 root 7r FIFO 0,7 8854 pipe
vmware-vm 4066 root 8w FIFO 0,7 8854 pipe
vmware-vm 4066 root 9r FIFO 0,7 8855 pipe
vmware-vm 4066 root 10w FIFO 0,7 8855 pipe
vmware-vm 4066 root 11r FIFO 0,7 8856 pipe
vmware-vm 4066 root 12w FIFO 0,7 8856 pipe
vmware-vm 4066 root 13r FIFO 0,7 8857 pipe
vmware-vm 4066 root 14w FIFO 0,7 8857 pipe
vmware-vm 4066 root 15r FIFO 0,7 8858 pipe
vmware-vm 4066 root 16w FIFO 0,7 8858 pipe
vmware-vm 4066 root 17u REG 253,0 20542 17514528
/var/lib/vmware/Virtual Machines/proxy/vmware.log
vmware-vm 4066 root 18r FIFO 0,7 8865 pipe
vmware-vm 4066 root 19w FIFO 0,7 8865 pipe
vmware-vm 4066 root 20r FIFO 0,7 8866 pipe
vmware-vm 4066 root 21w FIFO 0,7 8866 pipe
vmware-vm 4066 root 22r FIFO 0,7 8873 pipe
vmware-vm 4066 root 23w FIFO 0,7 8873 pipe
vmware-vm 4066 root 24r FIFO 0,7 8875 pipe
vmware-vm 4066 root 25w FIFO 0,7 8875 pipe
vmware-vm 4066 root 26u unix 0x00000100735bf080 8917
/var/run/vmware/root/4066/vmx-vmdb-fd
vmware-vm 4066 root 27u unix 0x000001007359e6c0 8920 socket
vmware-vm 4066 root 28r FIFO 0,7 35880 pipe
vmware-vm 4066 root 29r FIFO 0,7 35873 pipe
vmware-vm 4066 root 30r FIFO 0,7 35882 pipe
vmware-vm 4066 root 31w FIFO 0,7 35882 pipe
vmware-vm 4066 root 32r FIFO 0,7 8933 pipe
vmware-vm 4066 root 34u unix 0x0000010073616640 8883
/var/run/vmware/root/4066/servercontrol-fd
vmware-vm 4066 root 35u unix 0x0000010073616c40 8885
/var/run/vmware/root/4066/control-fd
vmware-vm 4066 root 36u unix 0x00000100735bf980 8887
/var/run/vmware/root/4066/ui-fd
vmware-vm 4066 root 37u CHR 10,165 7872 /dev/vmmon
vmware-vm 4066 root 38r FIFO 0,7 8889 pipe
vmware-vm 4066 root 39w FIFO 0,7 8889 pipe
vmware-vm 4066 root 40r FIFO 0,7 8889 pipe
vmware-vm 4066 root 41r FIFO 0,7 9050 pipe
vmware-vm 4066 root 42w FIFO 0,7 9050 pipe
vmware-vm 4066 root 43u unix 0x00000100735bf380 8891
/var/run/vmware/root/4066/mks-fd
vmware-vm 4066 root 45u CHR 10,165 7872 /dev/vmmon
vmware-vm 4066 root 46w FIFO 0,7 8933 pipe
vmware-vm 4066 root 47r FIFO 0,7 8934 pipe
vmware-vm 4066 root 48w FIFO 0,7 8934 pipe
vmware-vm 4066 root 49r FIFO 0,7 8935 pipe
vmware-vm 4066 root 50w FIFO 0,7 8935 pipe
vmware-vm 4066 root 51r FIFO 0,7 8936 pipe
vmware-vm 4066 root 52w FIFO 0,7 8936 pipe
vmware-vm 4066 root 53u unix 0x0000010073585680 9057
/var/run/vmware/root/4066/remoteDevice-fd
vmware-vm 4066 root 166u unix 0x00000100735bfc80 8949 socket
vmware-vm 4066 root 181u CHR 10,165 7872 /dev/vmmon
vmware-vm 4066 root 182u CHR 10,165 7872 /dev/vmmon
vmware-vm 4066 root 185r FIFO 0,7 35880 pipe
vmware-vm 4066 root 186w FIFO 0,7 35880 pipe
Output from sysrq t: (I don't know how to read this.)
SysRq : Show State
sibling
task PC pid father child younger older
init S 000000000000000b 0 1 0 2 (NOTLB)
0000010076451d78 0000000000000002 0000010076451da8 ffffffff801d4bf5
ffffffffffffffff ffffffff00000000 000000d000000000 0000000100000246
00000100764237f0 00000000000007e1
Call Trace:<ffffffff801d4bf5>{selinux_inode_permission+0}
<ffffffff80140090>{__mod_timer+293}
<ffffffff8030cbbb>{schedule_timeout+367}
<ffffffff80140aba>{process_timeout+0}
<ffffffff8018c4c3>{do_select+939} <ffffffff8018c05d>{__pollwait+0}
<ffffffff8018c842>{sys_select+820} <ffffffff8011026a>{system_call+126}
..... SNIP ....
acpid S 000001007325bac0 0 4038 1 4066 4028 (NOTLB)
000001006b4bdd78 0000000000000006 000001006c32b300 000000000000007a
0000010072c374c0 000000006b4bdd30 0000010001020a80 000000000000000e
000001006ba1b030 000000000001f24e
Call Trace:<ffffffff8030cb2c>{schedule_timeout+224}
<ffffffff8018c4c3>{do_select+939}
<ffffffff8018c05d>{__pollwait+0} <ffffffff8018c842>{sys_select+820}
<ffffffff801940e8>{dnotify_parent+34} <ffffffff8011026a>{system_call+126}
vmware-vmx D 0000000000000000 0 4066 1 4090 4038 (NOTLB)
000001006b6079f8 0000000000000006 ffffffffffffffff ffffffff00000069
0000000000000002 00000000a02349c6 0000010001028a80 00000001a023ba48
000001007300e030 000000000016a770
Call Trace:<ffffffffa022bc9c>{:vmmon:LinuxDriverError+32}
<ffffffff8030c2dd>{wait_for_completion+167}
<ffffffff801341cc>{default_wake_function+0}
<ffffffff80142abf>{group_send_sig_info+108}
<ffffffff801341cc>{default_wake_function+0}
<ffffffff80142b4f>{kill_proc_info+71}
<ffffffffa022e739>{:vmmon:HostIF_SetFastClockRate+353}
<ffffffffa022ff29>{:vmmon:Vmx86_SetHostClockRate+181}
<ffffffffa022f530>{:vmmon:Vmx86FreeAllVMResources+16}
<ffffffffa022fc9a>{:vmmon:Vmx86_ReleaseVM+62}
<ffffffffa022b20d>{:vmmon:LinuxDriver_Ioctl+249}
<ffffffff801628b0>{slab_destroy+124} <ffffffff801629f6>{free_block+285}
<ffffffffa006e726>{:jbd:do_get_write_access+1529}
<ffffffffa006e726>{:jbd:do_get_write_access+1529}
<ffffffffa006e726>{:jbd:do_get_write_access+1529}
<ffffffff8017e310>{__getblk+42}
<ffffffff8015ab70>{find_get_page+65}
<ffffffffa006de73>{:jbd:__journal_file_buffer+384}
<ffffffffa006e726>{:jbd:do_get_write_access+1529}
<ffffffffa006de73>{:jbd:__journal_file_buffer+384}
<ffffffff80134273>{__wake_up+54}
<ffffffffa006dad8>{:jbd:journal_stop+574}
<ffffffff8018fcd1>{dput+56} <ffffffff8017aca1>{__fput+253}
<ffffffff8030cd57>{__down_read+52}
<ffffffffa022a059>{:vmmon:LinuxDriver_Ioctl32_Handler+57}
<ffffffff8019f7e4>{compat_sys_ioctl+235}
<ffffffff80126257>{cstar_do_call+27}
cupsd S 0000000000000004 0 4090 1 4167 4066 (NOTLB)
000001006afe1d78 0000000000000006 ffffffff803d7680 0000000000000000
00000100736db180 000001006afe1d30 000000d000000000 0000000000000246
000001006af71030 000000000000052b
Call Trace:<ffffffff80140090>{__mod_timer+293}
<ffffffff8030cbbb>{schedule_timeout+367}
<ffffffff80140aba>{process_timeout+0} <ffffffff8018c4c3>{do_select+939}
<ffffffff8018c05d>{__pollwait+0} <ffffffff8018c842>{sys_select+820}
<ffffffff8018fd56>{dput+189} <ffffffff8011026a>{system_call+126}
..... SNIP ....
$ rpm -qa | grep -i vmw
VMware-server-1.0.2-39867
--
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
- -
- Jason Pyeron PD Inc. http://www.pdinc.us -
- Principal Consultant 10 West 24th Street #100 -
- +1 (443) 269-1555 x333 Baltimore, Maryland 21218 -
- -
-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
This message is copyright PD Inc, subject to license 20080407P00.
More information about the CentOS
mailing list