Hi All,
I run Linux server that hosts PHP/MySQL web-projects.
The problem is that it becomes irresponsive periodically, avg. load rapidly goes up to 100.0 and more and I have to do cold reboot.
Usually this sudden increase happens in 1-2 mins and it's very difficult to catch this moment in order to analyze why it happens.
After reboot load avg is stable around 1.0-2.0 during 2-3 days and then it suddenly increases again leading to complete freeze.
Last time I was lucky to do "ps ax" right before the freeze when load was around 14.0.
In 1 second it got completely freezed and I did reboot.
Could you please take a look at my "ps" dump below and suggest where the problem can be?
Is it mysql or apache, or anything else?
What I can do to prevent this in future, maybe some limitations in mysql or apache, or in Linux system?
Also here is some platform info:
OS: RH Fedora Core 11
CPU: Dual Xeon 2.8 with HT (4 cores)
RAM: 4 Gb
Apache MPM: prefork
Apache version: 2.2.15
MySQL engine: InnoDB
MySQL version: 5.1.46-1
Code
USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND
root 1 0.0 0.0 4076 252 ? Ss Jun20 0:08 /sbin/init
root 2 0.0 0.0 0 0 ? S< Jun20 0:00 [kthreadd]
root 3 0.0 0.0 0 0 ? S< Jun20 0:00 [migration/0]
root 4 0.0 0.0 0 0 ? S< Jun20 0:00 [ksoftirqd/0]
root 5 0.0 0.0 0 0 ? S< Jun20 0:00 [watchdog/0]
root 6 0.0 0.0 0 0 ? S< Jun20 0:00 [migration/1]
root 7 0.0 0.0 0 0 ? S< Jun20 0:00 [ksoftirqd/1]
root 8 0.0 0.0 0 0 ? S< Jun20 0:00 [watchdog/1]
root 9 0.0 0.0 0 0 ? S< Jun20 0:00 [migration/2]
root 10 0.0 0.0 0 0 ? S< Jun20 0:03 [ksoftirqd/2]
root 11 0.0 0.0 0 0 ? S< Jun20 0:00 [watchdog/2]
root 12 0.0 0.0 0 0 ? S< Jun20 0:00 [migration/3]
root 13 0.0 0.0 0 0 ? S< Jun20 0:01 [ksoftirqd/3]
root 14 0.0 0.0 0 0 ? S< Jun20 0:00 [watchdog/3]
root 15 0.0 0.0 0 0 ? S< Jun20 0:00 [events/0]
root 16 0.0 0.0 0 0 ? S< Jun20 0:00 [events/1]
root 17 0.0 0.0 0 0 ? S< Jun20 0:00 [events/2]
root 18 0.0 0.0 0 0 ? S< Jun20 0:00 [events/3]
root 19 0.0 0.0 0 0 ? S< Jun20 0:00 [cpuset]
root 20 0.0 0.0 0 0 ? S< Jun20 0:00 [khelper]
root 21 0.0 0.0 0 0 ? S< Jun20 0:00 [netns]
root 22 0.0 0.0 0 0 ? S< Jun20 0:00 [async/mgr]
root 23 0.0 0.0 0 0 ? S< Jun20 0:00 [kintegrityd/0]
root 24 0.0 0.0 0 0 ? S< Jun20 0:00 [kintegrityd/1]
root 25 0.0 0.0 0 0 ? S< Jun20 0:00 [kintegrityd/2]
root 26 0.0 0.0 0 0 ? S< Jun20 0:00 [kintegrityd/3]
root 27 0.0 0.0 0 0 ? S< Jun20 0:00 [kblockd/0]
root 28 0.0 0.0 0 0 ? S< Jun20 0:02 [kblockd/1]
root 29 0.0 0.0 0 0 ? S< Jun20 0:11 [kblockd/2]
root 30 0.0 0.0 0 0 ? S< Jun20 0:02 [kblockd/3]
root 31 0.0 0.0 0 0 ? S< Jun20 0:00 [kacpid]
root 32 0.0 0.0 0 0 ? S< Jun20 0:00 [kacpi_notify]
root 33 0.0 0.0 0 0 ? S< Jun20 0:00 [ata/0]
root 34 0.0 0.0 0 0 ? S< Jun20 0:00 [ata/1]
root 35 0.0 0.0 0 0 ? S< Jun20 0:00 [ata/2]
root 36 0.0 0.0 0 0 ? S< Jun20 0:00 [ata/3]
root 37 0.0 0.0 0 0 ? S< Jun20 0:00 [ata_aux]
root 38 0.0 0.0 0 0 ? S< Jun20 0:00 [ksuspend_usbd]
root 39 0.0 0.0 0 0 ? S< Jun20 0:00 [khubd]
root 40 0.0 0.0 0 0 ? S< Jun20 0:00 [kseriod]
root 41 0.0 0.0 0 0 ? S Jun20 0:00 [khungtaskd]
root 44 0.0 0.0 0 0 ? D< Jun20 3:00 [kswapd0]
root 45 0.0 0.0 0 0 ? S< Jun20 0:00 [aio/0]
root 46 0.0 0.0 0 0 ? S< Jun20 0:00 [aio/1]
root 47 0.0 0.0 0 0 ? S< Jun20 0:00 [aio/2]
root 48 0.0 0.0 0 0 ? S< Jun20 0:00 [aio/3]
root 49 0.0 0.0 0 0 ? S< Jun20 0:00 [crypto/0]
root 50 0.0 0.0 0 0 ? S< Jun20 0:00 [crypto/1]
root 51 0.0 0.0 0 0 ? S< Jun20 0:00 [crypto/2]
root 52 0.0 0.0 0 0 ? S< Jun20 0:00 [crypto/3]
root 62 0.0 0.0 0 0 ? S< Jun20 0:00 [scsi_eh_0]
root 63 0.0 0.0 0 0 ? S< Jun20 0:00 [scsi_eh_1]
root 68 0.0 0.0 0 0 ? S< Jun20 0:00 [kpsmoused]
root 69 0.0 0.0 0 0 ? S< Jun20 0:00 [kstriped]
root 70 0.0 0.0 0 0 ? S< Jun20 0:00 [ksnapd]
root 71 0.0 0.0 0 0 ? S< Jun20 0:00 [usbhid_resumer]
root 89 0.0 0.0 0 0 ? S< Jun20 0:00 [kjournald]
root 97 0.0 0.0 0 0 ? S< Jun20 0:00 [kauditd]
root 146 0.0 0.0 10452 376 ? S<s Jun20 0:00 /sbin/udevd
root 751 0.0 0.0 0 0 ? S< Jun20 0:00 [edac-poller]
root 858 0.0 0.0 0 0 ? S< Jun20 0:00 [kmpathd/0]
root 859 0.0 0.0 0 0 ? S< Jun20 0:00 [kmpathd/1]
root 860 0.0 0.0 0 0 ? S< Jun20 0:00 [kmpathd/2]
root 861 0.0 0.0 0 0 ? S< Jun20 0:00 [kmpathd/3]
root 863 0.0 0.0 0 0 ? S< Jun20 0:00 [kmpath_handlerd]
root 909 0.0 0.0 0 0 ? S< Jun20 0:00 [kjournald]
root 910 0.0 0.0 0 0 ? D< Jun20 2:09 [kjournald]
root 911 0.0 0.0 0 0 ? D< Jun20 0:57 [kjournald]
root 1297 0.0 0.0 92988 444 ? S<sl Jun20 0:06 auditd
root 1299 0.0 0.0 81912 436 ? S<sl Jun20 0:03 /sbin/audispd
root 1316 0.0 0.0 5992 104 ? Ss Jun20 0:00 /sbin/portreserve
root 1323 0.0 0.0 248452 592 ? Sl Jun20 0:36 rsyslogd
root 1335 0.0 0.0 6880 308 ? Ss Jun20 0:31 irqbalance
rpc 1342 0.0 0.0 18788 304 ? Ss Jun20 0:00 rpcbind
rpcuser 1355 0.0 0.0 10280 140 ? Ss Jun20 0:00 rpc.statd
root 1384 0.0 0.0 0 0 ? S< Jun20 0:00 [rpciod/0]
root 1386 0.0 0.0 0 0 ? S< Jun20 0:00 [rpciod/1]
root 1387 0.0 0.0 0 0 ? S< Jun20 0:00 [rpciod/2]
root 1388 0.0 0.0 0 0 ? S< Jun20 0:00 [rpciod/3]
root 1395 0.0 0.0 22924 84 ? Ss Jun20 0:00 rpc.idmapd
dbus 1408 0.0 0.0 21180 352 ? Ss Jun20 0:00 dbus-daemon
avahi 1417 0.0 0.0 25512 344 ? Ss Jun20 0:00 avahi-daemon:
avahi 1418 0.0 0.0 25388 56 ? Ss Jun20 0:00 avahi-daemon:
root 1442 0.0 0.0 3916 128 ? Ss Jun20 0:00 /usr/sbin/acpid
68 1450 0.0 0.0 28960 808 ? Ss Jun20 0:00 hald
root 1453 0.0 0.0 2064524 364 ? Ssl Jun20 0:00 /usr/sbin/console-kit-daemon
root 1454 0.0 0.0 17768 124 ? S Jun20 0:00 hald-runner
root 1545 0.0 0.0 19884 112 ? S Jun20 0:00 hald-addon-input:
root 1566 0.0 0.0 19896 112 ? S Jun20 0:00 /usr/libexec/hald-addon-cpufreq
68 1567 0.0 0.0 17480 132 ? S Jun20 0:00 hald-addon-acpi:
root 1580 0.0 0.0 93140 140 ? Ssl Jun20 0:00 pcscd
root 1595 0.0 0.0 63212 304 ? Ss Jun20 0:15 /usr/sbin/sshd
root 1603 0.0 0.0 21884 132 ? Ss Jun20 0:00 xinetd
ntp 1611 0.0 0.0 31940 528 ? Ss Jun20 0:00 ntpd
root 1626 0.0 0.0 67212 760 ? Ss Jun20 0:37 sendmail:
smmsp 1634 0.0 0.0 58816 420 ? Ss Jun20 0:00 sendmail:
root 1646 0.0 0.0 100292 320 ? Ss Jun20 0:01 crond
root 1657 0.0 0.0 16548 80 ? Ss Jun20 0:00 /usr/sbin/atd
root 1689 0.0 0.0 93928 136 ? S Jun20 0:00 /bin/sh
mysql 1859 57.5 30.2 4585880 1214076 ? Dl Jun20 2637:28 /usr/libexec/mysqld
root 1903 0.0 0.0 46620 96 ? Ss Jun20 0:00 /usr/sbin/vsftpd
root 1911 0.0 0.0 53648 432 ? Ss Jun20 0:00 /usr/sbin/dovecot
root 1912 0.0 0.0 54028 516 ? S Jun20 0:00 dovecot-auth
root 1952 0.0 0.0 18384 144 ? S Jun20 0:00 /usr/sbin/smartd
root 1965 0.0 0.0 3900 116 tty4 Ss+ Jun20 0:00 /sbin/mingetty
root 1966 0.0 0.0 3900 116 tty5 Ss+ Jun20 0:00 /sbin/mingetty
root 1967 0.0 0.0 3900 116 tty2 Ss+ Jun20 0:00 /sbin/mingetty
root 1968 0.0 0.0 3900 116 tty3 Ss+ Jun20 0:00 /sbin/mingetty
root 1969 0.0 0.0 3900 116 tty1 Ss+ Jun20 0:00 /sbin/mingetty
root 1970 0.0 0.0 3900 116 tty6 Ss+ Jun20 0:00 /sbin/mingetty
dovecot 1981 0.0 0.0 47584 168 ? S Jun20 0:00 imap-login
dovecot 1982 0.0 0.0 47584 168 ? S Jun20 0:00 imap-login
dovecot 1983 0.0 0.0 47584 168 ? S Jun20 0:00 imap-login
root 1989 0.0 0.1 482008 5984 ? Ss Jun20 1:11 /usr/local/apache/bin/httpd
root 12342 0.0 0.0 57432 308 ? Ss Jun21 0:00 squid
squid 12344 0.0 0.0 63484 2760 ? S Jun21 1:37 (squid)
squid 12346 0.0 0.0 22208 348 ? S Jun21 0:00 (unlinkd)
root 20646 0.0 0.0 71740 2072 ? S 15:23 0:00 sendmail:
root 21304 0.0 0.0 0 0 ? D 15:31 0:00 [pdflush]
root 21322 0.0 0.0 71740 2028 ? S 15:31 0:00 sendmail:
dovecot 22699 0.0 0.0 47568 1004 ? S 15:52 0:00 pop3-login
root 23156 0.0 0.0 72340 3804 ? S 15:56 0:00 sendmail:
root 23271 0.6 5.7 3063436 230932 ? Sl Jun20 28:27 /usr/lib/jvm/sun_jdk1.6.0_18/bin/java
root 23321 0.3 3.6 2814656 147324 ? Sl Jun20 14:17 /usr/lib/jvm/sun_jdk1.6.0_18/bin/java
root 23515 0.0 0.0 0 0 ? D 15:58 0:00 [pdflush]
root 23703 0.0 0.0 71540 2956 ? S 16:00 0:00 sendmail:
apache 24106 4.4 3.9 623712 157144 ? D 16:04 0:20 /usr/local/apache/bin/httpd
root 24171 0.0 0.0 54108 1540 ? S 10:20 0:00 dovecot-auth
apache 24199 1.0 3.2 607736 130980 ? D 16:05 0:04 /usr/local/apache/bin/httpd
apache 24202 1.7 3.1 604136 128472 ? D 16:05 0:07 /usr/local/apache/bin/httpd
apache 24410 1.6 3.6 621312 146072 ? D 16:07 0:05 /usr/local/apache/bin/httpd
apache 24427 0.5 0.4 490468 18896 ? S 16:07 0:01 /usr/local/apache/bin/httpd
apache 24429 2.3 4.0 628188 161900 ? D 16:07 0:06 /usr/local/apache/bin/httpd
apache 24458 1.4 3.4 615672 140508 ? D 16:07 0:04 /usr/local/apache/bin/httpd
apache 24460 1.6 3.5 618260 142796 ? D 16:07 0:04 /usr/local/apache/bin/httpd
apache 24517 0.4 0.3 488084 14992 ? D 16:07 0:01 /usr/local/apache/bin/httpd
apache 24671 1.6 3.3 610848 134648 ? D 16:09 0:02 /usr/local/apache/bin/httpd
apache 24679 0.7 0.3 489304 12916 ? S 16:09 0:01 /usr/local/apache/bin/httpd
apache 24682 1.8 3.8 633808 155344 ? D 16:09 0:03 /usr/local/apache/bin/httpd
apache 24686 1.3 2.8 593304 115448 ? D 16:09 0:02 /usr/local/apache/bin/httpd
apache 24688 1.3 3.0 604064 123704 ? D 16:09 0:02 /usr/local/apache/bin/httpd
apache 24718 2.0 3.6 623296 145928 ? D 16:09 0:02 /usr/local/apache/bin/httpd
apache 24719 4.4 8.4 820244 337916 ? D 16:09 0:06 /usr/local/apache/bin/httpd
apache 24722 2.0 3.4 615476 137760 ? D 16:09 0:02 /usr/local/apache/bin/httpd
apache 24753 1.9 3.2 615324 132256 ? D 16:10 0:02 /usr/local/apache/bin/httpd
apache 24809 3.3 0.3 487724 12072 ? S 16:11 0:02 /usr/local/apache/bin/httpd
apache 24810 0.5 0.2 486500 11084 ? S 16:11 0:00 /usr/local/apache/bin/httpd
apache 24816 0.1 0.2 483632 10412 ? S 16:11 0:00 /usr/local/apache/bin/httpd
apache 24817 0.2 0.2 483636 10412 ? S 16:11 0:00 /usr/local/apache/bin/httpd
apache 24825 0.1 0.2 484440 10216 ? S 16:11 0:00 /usr/local/apache/bin/httpd
apache 24828 0.1 0.2 483436 10552 ? S 16:11 0:00 /usr/local/apache/bin/httpd
apache 24829 0.2 0.2 484960 10704 ? D 16:11 0:00 /usr/local/apache/bin/httpd
apache 24831 0.2 0.2 484412 10644 ? S 16:11 0:00 /usr/local/apache/bin/httpd
apache 24832 0.1 0.2 483636 10340 ? S 16:11 0:00 /usr/local/apache/bin/httpd
root 24851 0.0 0.0 0 0 ? D 16:11 0:00 [pdflush]
root 24853 0.0 0.0 0 0 ? D 16:11 0:00 [pdflush]
apache 24855 0.2 0.2 484420 10508 ? D 16:11 0:00 /usr/local/apache/bin/httpd
root 24856 0.0 0.0 0 0 ? D 16:11 0:00 [pdflush]
apache 24861 0.4 0.2 483652 9400 ? S 16:12 0:00 /usr/local/apache/bin/httpd
apache 24863 0.3 0.2 484192 9372 ? D 16:12 0:00 /usr/local/apache/bin/httpd
apache 24864 0.0 0.1 482144 4684 ? S 16:12 0:00 /usr/local/apache/bin/httpd
apache 24866 0.5 0.2 483672 9684 ? S 16:12 0:00 /usr/local/apache/bin/httpd
apache 24867 0.6 0.2 483652 9604 ? S 16:12 0:00 /usr/local/apache/bin/httpd
apache 24868 0.1 0.1 483612 7496 ? D 16:12 0:00 /usr/local/apache/bin/httpd
apache 24869 0.1 0.1 483616 7464 ? D 16:12 0:00 /usr/local/apache/bin/httpd
root 24871 0.4 0.0 71540 3172 ? D 16:12 0:00 sendmail:
apache 24872 0.0 0.1 482144 4076 ? D 16:12 0:00 /usr/local/apache/bin/httpd
root 24873 0.5 0.0 72240 3280 ? D 16:12 0:00 sendmail:
root 24874 0.0 0.0 93816 1024 pts/1 R+ 16:12 0:00 ps
root 26426 0.0 0.0 98956 2004 ? Ss 10:43 0:01 sshd:
root 26434 0.0 0.2 100868 8092 pts/1 Ss 10:43 0:00 -bash
dovecot 27734 0.0 0.0 47568 1364 ? S 10:55 0:00 pop3-login
dovecot 29609 0.0 0.0 47568 1068 ? S 11:15 0:00 pop3-login
(Args of processes were removed for security reasons)
/etc/my.cnf (InnoDB part):
Code
innodb_buffer_pool_size = 2G
innodb_additional_mem_pool_size = 20M
innodb_log_file_size = 128M
innodb_log_buffer_size = 8M
innodb_lock_wait_timeout = 25
innodb_flush_log_at_trx_commit = 2
innodb_flush_method=O_DIRECT
wait_timeout = 20
#wait_timeout = 3600
max_connections = 300
/etc/httpd/extra/httpd-mpm.conf (prefork part):
Code
StartServers 5
MinSpareServers 5
MaxSpareServers 10
MaxClients 50
MaxRequestsPerChild 0
Thank you in advance!
Anatoliy,
Miraline
