[CentOS] CentOS won't shutdown ... or do anything else

Wed Mar 25 22:52:15 UTC 2009
hadrian at hushmail.com <hadrian at hushmail.com>

I started to have problems similar to ones described in the past on 
this list but could not find any kind of resolution. I did an 
lsmod, a mount command, and for fun, did an strace on shutdown to 
see where it is hanging, and an ltrace as well.

Any thoughts?

Module                  Size  Used by
parport_pc             28033  0
lp                     15661  0
parport                38153  2 parport_pc,lp
autofs4                26053  0
i2c_dev                14529  0
i2c_core               26305  1 i2c_dev
sg                     38369  0
sunrpc                144805  1
crc32c                  5953  8
libcrc32c               6721  1 crc32c
iscsi_sfnet            85073  38
scsi_transport_iscsi    12737  1 iscsi_sfnet
dm_multipath           23625  0
emcpdm                 39652  0
emcpgpx                20260  1 emcpdm
emcpmpx               158136  48
emcp                 1046804  3 emcpdm,emcpgpx,emcpmpx
emcplib                 6656  1 emcp
button                 10705  0
battery                12997  0
ac                      8901  0
md5                     8129  1
ipv6                  243809  32
joydev                 14465  0
uhci_hcd               33241  0
ehci_hcd               33353  0
i5000_edac             13121  0
edac_mc                29705  1 i5000_edac
bnx2                  141661  0
qla2300               130113  0
mptscsih                5569  0
ata_piix               19781  0
libata                106013  1 ata_piix
dm_snapshot            21221  0
dm_zero                 6337  0
dm_mirror              32453  0
ext3                  119753  6
jbd                    59865  1 ext3
dm_mod                 66921  11 
dm_multipath,dm_snapshot,dm_zero,dm_mirror
qla2322               141377  0
qla2400               234945  0
qla2xxx               175333  15 qla2300,qla2322,qla2400
scsi_transport_fc      12353  1 qla2xxx
megaraid_sas           38001  4
mptsas                 26069  1 mptscsih
mptfc                  12997  0
mptspi                 14417  1 mptscsih
mptscsi                44241  3 mptsas,mptfc,mptspi
mptbase                67873  4 mptsas,mptfc,mptspi,mptscsi
sd_mod                 20545  53
scsi_mod              120653  12 
sg,iscsi_sfnet,emcp,libata,qla2xxx,scsi_transport_fc,megaraid_sas,mp
tsas,mptfc,mptspi,mptscsi,sd_mod

/dev/mapper/VolGroup00-LogVol00 on / type ext3 (rw)
none on /proc type proc (rw)
none on /sys type sysfs (rw)
none on /dev/pts type devpts (rw,gid=5,mode=620)
usbfs on /proc/bus/usb type usbfs (rw)
/dev/sda2 on /boot type ext3 (rw)
none on /dev/shm type tmpfs (rw)
none on /proc/sys/fs/binfmt_misc type binfmt_misc (rw)
sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw)
/dev/emcpowerd on /EMC/SATA/AX4-5i/LUN0 type ext3 
(rw,_netdev,noatime)
/EMC/SATA/AX4-5i/LUN0/path/var/spool/postfix on /var/spool/postfix 
type ext3 (rw,bind,_netdev,noatime)
/dev/emcpowerk on /EMC/SATA/AX4-5i/LUN4 type ext3 
(rw,_netdev,noatime)
/EMC/SATA/AX4-5i/LUN4 on <path> type ext3 (rw,bind,_netdev,noatime)
/dev/emcpowerg on /EMC/SATA/AX4-5i/LUN5 type ext3 
(rw,_netdev,noatime)
/dev/emcpowerj on /EMC/SATA/AX4-5i/LUN6 type ext3 
(rw,_netdev,noatime)


execve("/sbin/shutdown", ["shutdown", "-r", "now"], [/* 29 vars 
*/]) = 0
uname({sys="Linux", node="host.dom", ...}) = 0
brk(0)                                  = 0x9030000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file 
or directory)
open("/etc/ld.so.cache", O_RDONLY)      = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=102323, ...}) = 0
old_mmap(NULL, 102323, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb7f4a000
close(3)                                = 0
open("/lib/tls/libc.so.6", O_RDONLY)    = 3
read(3, 
"\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\320\336"..., 512) 
= 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=1525004, ...}) = 0
old_mmap(0x3d9000, 1223900, PROT_READ|PROT_EXEC, 
MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x3d9000
old_mmap(0x4fe000, 16384, PROT_READ|PROT_WRITE, 
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x124000) = 0x4fe000
old_mmap(0x502000, 7388, PROT_READ|PROT_WRITE, 
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x502000
close(3)                                = 0
old_mmap(NULL, 4096, PROT_READ|PROT_WRITE, 
MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7f49000
mprotect(0x4fe000, 8192, PROT_READ)     = 0
mprotect(0x3d0000, 4096, PROT_READ)     = 0
set_thread_area({entry_number:-1 -> 6, base_addr:0xb7f49aa0, 
limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, 
limit_in_pages:1, seg_not_present:0, useable:1}) = 0
munmap(0xb7f4a000, 102323)              = 0
getuid32()                              = 0
geteuid32()                             = 0
setuid32(0)                             = 0
getuid32()                              = 0
brk(0)                                  = 0x9030000
brk(0x9051000)                          = 0x9051000
open("/var/run/shutdown.pid", O_RDONLY) = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=6, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -
1, 0) = 0xb7f62000
read(3, "30314\n", 4096)                = 6
close(3)                                = 0
munmap(0xb7f62000, 4096)                = 0
kill(30314, SIG_0)                      = -1 ESRCH (No such process)
unlink("/var/run/shutdown.pid")         = 0
umask(022)                              = 022
open("/var/run/shutdown.pid", O_WRONLY|O_CREAT|O_TRUNC, 0666) = 3
getpid()                                = 30317
fstat64(3, {st_mode=S_IFREG|0644, st_size=0, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -
1, 0) = 0xb7f62000
write(3, "30317\n", 6)                  = 6
close(3)                                = 0
munmap(0xb7f62000, 4096)                = 0
rt_sigaction(SIGQUIT, {SIG_IGN}, {SIG_DFL}, 8) = 0
rt_sigaction(SIGCHLD, {SIG_IGN}, {SIG_DFL}, 8) = 0
rt_sigaction(SIGHUP, {SIG_IGN}, {SIG_DFL}, 8) = 0
rt_sigaction(SIGTSTP, {SIG_IGN}, {SIG_DFL}, 8) = 0
rt_sigaction(SIGTTIN, {SIG_IGN}, {SIG_DFL}, 8) = 0
rt_sigaction(SIGTTOU, {SIG_IGN}, {SIG_DFL}, 8) = 0
rt_sigaction(SIGINT, {0x8049938, [], SA_RESTORER|SA_RESTART, 
0x400898}, NULL, 8) = 0
chdir("/")                              = 0
getuid32()                              = 0
socket(PF_FILE, SOCK_STREAM, 0)         = 3
fcntl64(3, F_GETFL)                     = 0x2 (flags O_RDWR)
fcntl64(3, F_SETFL, O_RDWR|O_NONBLOCK)  = 0
connect(3, {sa_family=AF_FILE, path="/var/run/nscd/socket"}, 110) = 
-1 ENOENT (No such file or directory)
close(3)                                = 0
socket(PF_FILE, SOCK_STREAM, 0)         = 3
fcntl64(3, F_GETFL)                     = 0x2 (flags O_RDWR)
fcntl64(3, F_SETFL, O_RDWR|O_NONBLOCK)  = 0
connect(3, {sa_family=AF_FILE, path="/var/run/nscd/socket"}, 110) = 
-1 ENOENT (No such file or directory)
close(3)                                = 0
open("/etc/nsswitch.conf", O_RDONLY)    = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=1623, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -
1, 0) = 0xb7f62000
read(3, "#\n# /etc/nsswitch.conf\n#\n# An ex"..., 4096) = 1623
read(3, "", 4096)                       = 0
close(3)                                = 0
munmap(0xb7f62000, 4096)                = 0
open("/etc/ld.so.cache", O_RDONLY)      = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=102323, ...}) = 0
old_mmap(NULL, 102323, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb7f4a000
close(3)                                = 0
open("/lib/libnss_files.so.2", O_RDONLY) = 3
read(3, 
"\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\20\33\0"..., 512) 
= 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=47468, ...}) = 0
old_mmap(NULL, 41608, PROT_READ|PROT_EXEC, 
MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x2fa000
old_mmap(0x303000, 8192, PROT_READ|PROT_WRITE, 
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x8000) = 0x303000
close(3)                                = 0
mprotect(0x303000, 4096, PROT_READ)     = 0
munmap(0xb7f4a000, 102323)              = 0
open("/etc/passwd", O_RDONLY)           = 3
fcntl64(3, F_GETFD)                     = 0
fcntl64(3, F_SETFD, FD_CLOEXEC)         = 0
fstat64(3, {st_mode=S_IFREG|0644, st_size=2778, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -
1, 0) = 0xb7f62000
read(3, "root:x:0:0:root:/root:/bin/bash\n"..., 4096) = 2778
close(3)                                = 0
munmap(0xb7f62000, 4096)                = 0
ioctl(0, SNDCTL_TMR_TIMEBASE or TCGETS, {B38400 opost isig icanon 
echo ...}) = 0
readlink("/proc/self/fd/0", "/dev/pts/4", 4095) = 10
time(NULL)                              = 1238017686
open("/etc/localtime", O_RDONLY)        = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=1017, ...}) = 0
fstat64(3, {st_mode=S_IFREG|0644, st_size=1017, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -
1, 0) = 0xb7f62000
read(3, 
"TZif\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\4\0\0\0\4\0"..., 4096) 
= 1017
close(3)                                = 0
munmap(0xb7f62000, 4096)                = 0
clone(child_stack=0, 
flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, 
child_tidptr=0xb7f49ae8) = 30318
time(NULL)                              = 1238017686
stat64("/etc/localtime", {st_mode=S_IFREG|0644, st_size=1017, ...}) 
= 0
stat64("/etc/localtime", {st_mode=S_IFREG|0644, st_size=1017, ...}) 
= 0
stat64("/etc/localtime", {st_mode=S_IFREG|0644, st_size=1017, ...}) 
= 0
socket(PF_FILE, SOCK_DGRAM, 0)          = 3
fcntl64(3, F_SETFD, FD_CLOEXEC)         = 0
connect(3, {sa_family=AF_FILE, path="/dev/log"}, 16) = 0
send(3, "<13>Mar 25 14:48:06 shutdown: sh"..., 61, MSG_NOSIGNAL 
<unfinished ...>

_start(0xbff00640, 0xbff00649, 0xbff0064c, 0, 0xbff00650 
<unfinished ...>
__libc_start_main(0x8049ffc, 3, 0xbfe01174, 0x804acac, 0x804ad00 
<unfinished ...>
getuid()                                                            
                             = 0
geteuid()                                                           
                             = 0
setuid(0)                                                           
                             = 0
getuid()                                                            
                             = 0
getopt(3, 0xbfe01174, "acqkrhnfFyt:g:i:")                           
                             = 114
getopt(3, 0xbfe01174, "acqkrhnfFyt:g:i:")                           
                             = -1
fopen("/var/run/shutdown.pid", "r")                                 
                             = 0x94ad008
fscanf(0x94ad008, 0x804adf0, 0xbfe00eac, 0, 81)                     
                             = 1
fclose(0x94ad008)                                                   
                             = 0
kill(30365, 0)                                                      
                             = -1
unlink("/var/run/shutdown.pid")                                     
                             = 0
umask(022)                                                          
                             = 022
fopen("/var/run/shutdown.pid", "w")                                 
                             = 0x94ad008
getpid()                                                            
                             = 30451
fprintf(0x94ad008, "%d\n", 30451)                                   
                             = 6
fclose(0x94ad008)                                                   
                             = 0
signal(3, 0x1)                                                      
                             = NULL
signal(17, 0x1)                                                     
                             = NULL
signal(1, 0x1)                                                      
                             = NULL
signal(20, 0x1)                                                     
                             = NULL
signal(21, 0x1)                                                     
                             = NULL
signal(22, 0x1)                                                     
                             = NULL
sigaction(2, 0xbfe00eb0, NULL)                                      
                             = 0
chdir("/")                                                          
                             = 0
strchr("0", ':')                                                    
                             = NULL
__strtol_internal("0", NULL, 10)                                    
                             = 0
strncat(0xbfe00cb0, 0x804c960, 319, 0, 0)                           
                             = 0xbfe00cb0
snprintf("\rThe system is going down for re"..., 320, "\rThe system 
is going down %s NO"..., "for reboot") = 43
getuid()                                                            
                             = 0
getpwuid(0, 0x46478f, 0xbfe00c98, 0x80494e9, 0x3d0fd4)              
                             = 0x5020bc
strncat(0x804c840, 0x94ad008, 31, 0xbfe00cb1, 0x3d0fd4)             
                             = 0x804c840
ttyname(0)                                                          
                             = "/dev/pts/4"
sprintf("(pts/4) ", "(%.28s) ", "pts/4")                            
                             = 8
time(0xbfe00af4)                                                    
                             = 1238018002
ctime(0xbfe00af4)                                                   
                             = "Wed Mar 25 14:53:22 2009\n"
snprintf("\007\r\nBroadcast message from root ("..., 81, 
"\007\r\nBroadcast message from %"..., "root", "(pts/4) ") = 70
fork(
Broadcast message from root (pts/4) (Wed Mar 25 14:53:22 2009):

The system is going down for reboot NOW!
)                                                                   
                        = 30452
openlog("shutdown", 4, 8)                                           
                             = <void>
syslog(5, "shutting down for system reboot"

--
Become a medical transcriptionist at home, at your own pace.
 http://tagline.hushmail.com/fc/BLSrjkqfMmeLwBnPlPu3jmrCxuyZBMvKFibhY6JYoa8KvP8ydneFuVcu33i/