Hey guys, Well it took a little while for me to be able to reproduce this. It seems that this problem is intermittent and sporadic. But I tried running a sh -x /etc/init.d/httpd restart command once I reallized I had another incident of this and this is what I saw as the output: + . /etc/rc.d/init.d/functions ++ TEXTDOMAIN=initscripts ++ umask 022 ++ PATH=/sbin:/usr/sbin:/bin:/usr/bin ++ export PATH ++ '[' -z '' ']' ++ COLUMNS=80 ++ '[' -z '' ']' +++ /sbin/consoletype ++ CONSOLETYPE=pty ++ '[' -f /etc/sysconfig/i18n -a -z '' ']' ++ . /etc/profile.d/lang.sh +++ sourced=0 +++ '[' -z '' -a -n en_US.UTF-8 ']' +++ sourced=1 +++ '[' -n '' ']' +++ '[' 1 = 1 ']' +++ '[' -n en_US.UTF-8 ']' +++ export LANG +++ '[' -n '' ']' +++ unset LC_ADDRESS +++ '[' -n '' ']' +++ unset LC_CTYPE +++ '[' -n '' ']' +++ unset LC_COLLATE +++ '[' -n '' ']' +++ unset LC_IDENTIFICATION +++ '[' -n '' ']' +++ unset LC_MEASUREMENT +++ '[' -n '' ']' +++ unset LC_MESSAGES +++ '[' -n '' ']' +++ unset LC_MONETARY +++ '[' -n '' ']' +++ unset LC_NAME +++ '[' -n '' ']' +++ unset LC_NUMERIC +++ '[' -n '' ']' +++ unset LC_PAPER +++ '[' -n '' ']' +++ unset LC_TELEPHONE +++ '[' -n '' ']' +++ unset LC_TIME +++ '[' -n '' ']' +++ unset LC_ALL +++ '[' -n '' ']' +++ unset LANGUAGE +++ '[' -n '' ']' +++ unset LINGUAS +++ '[' -n '' ']' +++ unset _XKB_CHARSET +++ consoletype=pty +++ '[' -z pty ']' +++ '[' -n '' ']' +++ '[' -n '' ']' +++ '[' -n en_US.UTF-8 ']' +++ case $LANG in +++ '[' screen = linux ']' +++ unset SYSFONTACM SYSFONT +++ unset sourced +++ unset langfile ++ '[' -z '' ']' ++ '[' -f /etc/sysconfig/init ']' ++ . /etc/sysconfig/init +++ BOOTUP=color +++ GRAPHICAL=yes +++ RES_COL=60 +++ MOVE_TO_COL='echo -en \033[60G' +++ SETCOLOR_SUCCESS='echo -en \033[0;32m' +++ SETCOLOR_FAILURE='echo -en \033[0;31m' +++ SETCOLOR_WARNING='echo -en \033[0;33m' +++ SETCOLOR_NORMAL='echo -en \033[0;39m' +++ LOGLEVEL=3 +++ PROMPT=yes +++ AUTOSWAP=no ++ '[' pty = serial ']' ++ '[' color '!=' verbose ']' ++ INITLOG_ARGS=-q ++ __sed_discard_ignored_files='/\(~\|\.bak\|\.orig\|\.rpmnew\|\.rpmorig\|\.rpmsave\)$/d' + '[' -f /etc/sysconfig/httpd ']' + . /etc/sysconfig/httpd + HTTPD_LANG=C + INITLOG_ARGS= + apachectl=/usr/sbin/apachectl + httpd=/usr/sbin/httpd + prog=httpd + pidfile=/var/run/httpd/httpd.pid + lockfile=/var/lock/subsys/httpd + RETVAL=0 + case "$1" in + stop + echo -n 'Stopping httpd: ' Stopping httpd: + killproc -p /var/run/httpd/httpd.pid -d 10 /usr/sbin/httpd + local RC killlevel= base pid pid_file= delay + RC=0 + delay=3 + '[' 5 -eq 0 ']' + '[' -p = -p ']' + pid_file=/var/run/httpd/httpd.pid + shift 2 + '[' -d = -d ']' + delay=10 + shift 2 + '[' -n '' ']' + base=httpd + __pids_var_run /usr/sbin/httpd /var/run/httpd/httpd.pid + local base=httpd + local pid_file=/var/run/httpd/httpd.pid + pid= + '[' -f /var/run/httpd/httpd.pid ']' + return 3 + '[' -z /var/run/httpd/httpd.pid -a -z '' ']' + '[' -n '' ']' + '[' -n '' -a -n '' ']' + failure 'httpd shutdown' + local rc=1 + '[' color '!=' verbose -a -z '' ']' + echo_failure + '[' color = color ']' + echo -en '\033[60G' [60G+ echo -n '[' [+ '[' color = color ']' + echo -en '\033[0;31m' [0;31m+ echo -n FAILED FAILED+ '[' color = color ']' + echo -en '\033[0;39m' [0;39m+ echo -n ']' ]+ echo -ne '\r' + return 1 + '[' -x /usr/bin/rhgb-client ']' + return 1 + RC=0 + '[' -z '' ']' + rm -f /var/run/httpd/httpd.pid + return 0 + RETVAL=0 + echo + '[' 0 = 0 ']' + rm -f /var/lock/subsys/httpd /var/run/httpd/httpd.pid + start + echo -n 'Starting httpd: ' Starting httpd: + LANG=C + daemon --pidfile=/var/run/httpd/httpd.pid /usr/sbin/httpd + local gotbase= force= nicelevel corelimit + local pid base= user= nice= bg= pid_file= + nicelevel=0 + '[' --pidfile=/var/run/httpd/httpd.pid '!=' -pidfile=/var/run/httpd/httpd.pid ']' + case $1 in + pid_file=/var/run/httpd/httpd.pid + shift + '[' /usr/sbin/httpd '!=' /usr/sbin/httpd ']' + '[' -z '' ']' + base=httpd + __pids_var_run httpd /var/run/httpd/httpd.pid + local base=httpd + local pid_file=/var/run/httpd/httpd.pid + pid= + '[' -f /var/run/httpd/httpd.pid ']' + return 3 + '[' -n '' -a -z '' ']' + corelimit='ulimit -S -c 0' + '[' -n '' ']' + '[' color = verbose -a -z '' ']' + '[' -z '' ']' + /bin/bash -c 'ulimit -S -c 0 >/dev/null 2>&1 ; /usr/sbin/httpd' (98)Address already in use: make_sock: could not bind to address [::]:80 (98)Address already in use: make_sock: could not bind to address 0.0.0.0:80 no listening sockets available, shutting down Unable to open logs + '[' 1 -eq 0 ']' + failure 'httpd startup' + local rc=1 + '[' color '!=' verbose -a -z '' ']' + echo_failure + '[' color = color ']' + echo -en '\033[60G' [60G+ echo -n '[' [+ '[' color = color ']' + echo -en '\033[0;31m' [0;31m+ echo -n FAILED FAILED+ '[' color = color ']' + echo -en '\033[0;39m' [0;39m+ echo -n ']' ]+ echo -ne '\r' + return 1 + '[' -x /usr/bin/rhgb-client ']' + return 1 + RETVAL=1 + echo + '[' 1 = 0 ']' + return 1 + exit 1 Not really sure how to interpret that, unfortunately. However looked for the pid file for apache and noticed that it DOESN'T EXIST! [root at beta:~] #ls -l /var/run/httpd/ total 0 Well, that would explain why the init script isn';t able to kill the process. Maybe puppet is doing something weird with that pid file? I don't really know offhand, but I guess I will have to investigate that. Thanks for all your input. Tim On Tue, Mar 4, 2014 at 5:07 PM, Les Mikesell <lesmikesell at gmail.com> wrote: > On Tue, Mar 4, 2014 at 3:42 PM, Tim Dunphy <bluethundr at gmail.com> wrote: > > Hey thanks for the input! > > > > Are you sure all the instances were started by the stock initscript > >> initially ... it sounds like something is killing or creating apache > >> instances that are in /var/run/httpd/ > >> That normally happens if httpd is started manually by another script or > >> user process rather than the one in /etc/init.d/ > > > > I meant something else is starting httpd and NOT putting the info in > > > > > > Absolutely positive that apache is not being started any other way than > the > > init script in /etc/init.d. > > > > So does the process id in /var/run/httpd/httpd.pid belong to the > master apache instance, and does killing it kill all of them? > > -- > Les Mikesell > lesmikesell at gmail.com > _______________________________________________ > CentOS mailing list > CentOS at centos.org > http://lists.centos.org/mailman/listinfo/centos > -- GPG me!! gpg --keyserver pool.sks-keyservers.net --recv-keys F186197B