9 ÄÅËÁÂÒÑ 2010šÇ. 1:19 ÐÏÌØÚÏ×ÁÔÅÌØ -=HaRius=-
# vmstat -z
ITEM š š š š š š š š š š SIZE š š LIMIT š š šUSED š š šFREE šREQUESTS šFAILURES
UMA Kegs: š š š š š š š š 208, š š š š0, š š š 96, š š š š6, š š š 96, š š š š0
UMA Zones: š š š š š š š š704, š š š š0, š š š 96, š š š š4, š š š 96, š š š š0
UMA Slabs: š š š š š š š š568, š š š š0, š š18921, š š š š7, š š20091, š š š š0
UMA RCntSlabs: š š š š š š568, š š š š0, š š16896, š š š š2, š š16896, š š š š0
UMA Hash: š š š š š š š š 256, š š š š0, š š š š0, š š š 15, š š š š3, š š š š0
16 Bucket: š š š š š š š š152, š š š š0, š š š165, š š š 10, š š š172, š š š š0
32 Bucket: š š š š š š š š280, š š š š0, š š š298, š š š 10, š š š299, š š š š0
64 Bucket: š š š š š š š š536, š š š š0, š š š406, š š š š0, š š š425, š š š270
128 Bucket: š š š š š š š1048, š š š š0, š š 3293, š š š š7, 31842996, š š š 25
VM OBJECT: š š š š š š š š216, š š š š0, š š 5805, š š 1251, š 290347, š š š š0
MAP: š š š š š š š š š š š232, š š š š0, š š š š7, š š š 25, š š š š7, š š š š0
KMAP ENTRY: š š š š š š š 120, š 413013, š š š 56, š š š719, š š 9240, š š š š0
MAP ENTRY: š š š š š š š š120, š š š š0, š š 1850, š š 2025, š 955226, š š š š0
DP fakepg: š š š š š š š š120, š š š š0, š š š š0, š š š š0, š š š š0, š š š š0
SG fakepg: š š š š š š š š120, š š š š0, š š š š0, š š š š0, š š š š0, š š š š0
mt_zone: š š š š š š š š 2056, š š š š0, š š š207, š š š š8, š š š207, š š š š0
16: š š š š š š š š š š š š16, š š š š0, š š 2754, š š 1782, š4089333, š š š š0
32: š š š š š š š š š š š š32, š š š š0, š š 3706, š š 2253, š š28850, š š š š0
64: š š š š š š š š š š š š64, š š š š0, š š 5531, š š 2701, š 201069, š š š š0
128: š š š š š š š š š š š128, š š š š0, š š 7298, š š 2011, š š62918, š š š š0
256: š š š š š š š š š š š256, š š š š0, š š 2165, š š 1285, š š59492, š š š š0
512: š š š š š š š š š š š512, š š š š0, š š 1006, š š š863, š š28438, š š š š0
1024: š š š š š š š š š š1024, š š š š0, š š š 84, š š š556, š š19245, š š š š0
2048: š š š š š š š š š š2048, š š š š0, š š š 75, š š š323, š š 1494, š š š š0
4096: š š š š š š š š š š4096, š š š š0, š š š340, š š š678, š š16753, š š š š0
Files: š š š š š š š š š š 80, š š š š0, š š21987, š š56133, š4872774, š š š š0
TURNSTILE: š š š š š š š š136, š š š š0, š š š737, š š š163, š š š737, š š š š0
umtx pi: š š š š š š š š š 96, š š š š0, š š š š0, š š š š0, š š š š0, š š š š0
MAC labels: š š š š š š š š40, š š š š0, š š š š0, š š š š0, š š š š0, š š š š0
PROC: š š š š š š š š š š1120, š š š š0, š š š 70, š š š410, š š12414, š š š š0
THREAD: š š š š š š š š š 912, š š š š0, š š š573, š š š163, š š š581, š š š š0
SLEEPQUEUE: š š š š š š š š64, š š š š0, š š š737, š š š607, š š š737, š š š š0
VMSPACE: š š š š š š š š š392, š š š š0, š š š 51, š š š529, š š12395, š š š š0
cpuset: š š š š š š š š š š72, š š š š0, š š š š2, š š š 98, š š š š2, š š š š0
audit_record: š š š š š š 952, š š š š0, š š š š0, š š š š0, š š š š0, š š š š0
mbuf_packet: š š š š š š š256, š š š š0, š š š š0, š š 1408, š š 8528, š š š š0
mbuf: š š š š š š š š š š 256, š š š š0, š š 1150, š š33022, 140995574, š š š š0
mbuf_cluster: š š š š š š2048, š 262144, š š 2003, š š31789, 107340535, š š10380
mbuf_jumbo_page: š š š š 4096, š š16896, š š š š0, š š š š0, š š š š0, š š š š0
mbuf_jumbo_9k: š š š š š 9216, š š 8448, š š š š0, š š š š0, š š š š0, š š š š0
mbuf_jumbo_16k: š š š š 16384, š š 4224, š š š š0, š š š š0, š š š š0, š š š š0
mbuf_ext_refcnt: š š š š š š4, š š š š0, š š š 42, š š 2142, š 132034, š š š š0
ttyinq: š š š š š š š š š 160, š š š š0, š š š165, š š š171, š š š600, š š š š0
ttyoutq: š š š š š š š š š256, š š š š0, š š š 88, š š š122, š š š320, š š š š0
g_bio: š š š š š š š š š š232, š š š š0, š š š š0, š š š976, š 128888, š š š š0
ata_request: š š š š š š š312, š š š š0, š š š š0, š š š912, š š32536, š š š š0
ata_composite: š š š š š š336, š š š š0, š š š š0, š š š š0, š š š š0, š š š š0
VNODE: š š š š š š š š š š472, š š š š0, š š 4458, š š š806, š š 4770, š š š š0
VNODEPOLL: š š š š š š š š112, š š š š0, š š š š2, š š š 97, š š š š2, š š š š0
S VFS Cache: š š š š š š š108, š š š š0, š š 4658, š š š589, š 119262, š š š š0
L VFS Cache: š š š š š š š328, š š š š0, š š š š0, š š š 48, š š š š6, š š š š0
NAMEI: š š š š š š š š š 1024, š š š š0, š š š š0, š š š288, š1887311, š š š š0
DIRHASH: š š š š š š š š 1024, š š š š0, š š 1116, š š š128, š š 1116, š š š š0
NFSMOUNT: š š š š š š š š 608, š š š š0, š š š š0, š š š š0, š š š š0, š š š š0
NFSNODE: š š š š š š š š š648, š š š š0, š š š š0, š š š š0, š š š š0, š š š š0
pipe: š š š š š š š š š š 728, š š š š0, š š š š6, š š š354, š š 8010, š š š š0
ksiginfo: š š š š š š š š 112, š š š š0, š š š470, š š š586, š š š470, š š š š0
itimer: š š š š š š š š š 344, š š š š0, š š š š0, š š š š0, š š š š0, š š š š0
KNOTE: š š š š š š š š š š120, š š š š0, š š20377, š š56348, 13244547, š š š š0
socket: š š š š š š š š š 680, š 102402, š š29980, š š71828, 29414249, š š š š0
unpcb: š š š š š š š š š š240, š 102400, š š š 47, š š š321, š š š552, š š š š0
ipq: š š š š š š š š š š š 56, š š 8253, š š š š0, š š š š0, š š š š0, š š š š0
udp_inpcb: š š š š š š š š336, š 102410, š š š š2, š š š328, š š 2212, š š š š0
udpcb: š š š š š š š š š š 16, š 102480, š š š š2, š š 1846, š š 2212, š š š š0
tcp_inpcb: š š š š š š š š336, š 102410, š š49130, š š53280, š4021759, 25388188
tcpcb: š š š š š š š š š š880, š 102400, š š29931, š š71825, š4021759, š š š š0
tcptw: š š š š š š š š š š 72, š š20500, š š19199, š š 1301, š 267339, š3289061
syncache: š š š š š š š š 144, š 102414, š š 1613, š š18927, 18078696, š š š š0
hostcache: š š š š š š š š136, š š15372, š š 1117, š š š479, š š 1117, š š š š0
tcpreass: š š š š š š š š š40, š š16464, š š š 53, š š š787, š š 4682, š š š š0
sackhole: š š š š š š š š š32, š š š š0, š š š š0, š š 1818, š š 9095, š š š š0
ripcb: š š š š š š š š š š336, š 102410, š š š š0, š š š220, š š 1537, š š š š0
rtentry: š š š š š š š š š200, š š š š0, š š š š6, š š š 51, š š š š6, š š š š0
pfsrctrpl: š š š š š š š š152, š š š š0, š š š š0, š š š š0, š š š š0, š š š š0
pfrulepl: š š š š š š š š 912, š š š š0, š š š š0, š š š š0, š š š š0, š š š š0
pfstatepl: š š š š š š š š392, š š10000, š š š š0, š š š š0, š š š š0, š š š š0
pfaltqpl: š š š š š š š š 240, š š š š0, š š š š0, š š š š0, š š š š0, š š š š0
pfpooladdrpl: š š š š š š š88, š š š š0, š š š š0, š š š š0, š š š š0, š š š š0
pfrktable: š š š š š š š 1296, š š š š0, š š š š0, š š š š0, š š š š0, š š š š0
pfrkentry: š š š š š š š š216, š š š š0, š š š š0, š š š š0, š š š š0, š š š š0
pfrkentry2: š š š š š š š 216, š š š š0, š š š š0, š š š š0, š š š š0, š š š š0
pffrent: š š š š š š š š š 32, š š 5050, š š š š0, š š š š0, š š š š0, š š š š0
pffrag: š š š š š š š š š š80, š š š š0, š š š š0, š š š š0, š š š š0, š š š š0
pffrcache: š š š š š š š š 80, š š10035, š š š š0, š š š š0, š š š š0, š š š š0
pffrcent: š š š š š š š š š24, š š50022, š š š š0, š š š š0, š š š š0, š š š š0
pfstatescrub: š š š š š š š40, š š š š0, š š š š0, š š š š0, š š š š0, š š š š0
pfiaddrpl: š š š š š š š š120, š š š š0, š š š š0, š š š š0, š š š š0, š š š š0
pfospfen: š š š š š š š š 112, š š š š0, š š š š0, š š š š0, š š š š0, š š š š0
pfosfp: š š š š š š š š š š40, š š š š0, š š š š0, š š š š0, š š š š0, š š š š0
IPFW dynamic rule: š š š š120, š š š š0, š š š š0, š š š š0, š š š š0, š š š š0
selfd: š š š š š š š š š š 56, š š š š0, š š š236, š š 1024, š š54606, š š š š0
ip4flow: š š š š š š š š š 56, š š 4158, š š 3642, š š š516, š 120462, š3152043
ip6flow: š š š š š š š š š 80, š š 4140, š š š š0, š š š š0, š š š š0, š š š š0
SWAPMETA: š š š š š š š š 288, š 116519, š š š š0, š š š š0, š š š š0, š š š š0
Mountpoints: š š š š š š š752, š š š š0, š š š š7, š š š 23, š š š š7, š š š š0
FFS inode: š š š š š š š š168, š š š š0, š š 4405, š š š611, š š 4713, š š š š0
FFS1 dinode: š š š š š š š128, š š š š0, š š š š0, š š š š0, š š š š0, š š š š0
FFS2 dinode: š š š š š š š256, š š š š0, š š 4405, š š š515, š š 4711, š š š š0
# sysctl hw.intr_storm_threshold
hw.intr_storm_threshold: 4000
ÓÅÔÅ×ÕÈÁ igb ÎÁ šËÁËÏÊ ÔÏ supermicro ÐÌÁÔÆÏÒÍÅ.
> reset_timedout_connection on;š
ËÏÎÅÞÎÏ ×ËÌÀÞÅÎ
> èÏÔÑ 90k - ÜÔÏ × ÏÂÝÅÍ ÎÅ ÍÎÏÇÏ.š
× 9 ÒÁÚ ÂÏÌØÛÅ ÏÂÙÞÎÏÇÏ =(
sysctl kern.ipc.nmbclusters=262144
Õ×ÅÌÉÞÉÌ.
2010/12/9 Maxim Dounin
<mdounin@xxxxxxxxxx>
Hello!
On Thu, Dec 09, 2010 at 12:07:32AM +0300, -=HaRius=- wrote:
> ÎÁ×ÅÒÎÏÅ ÐÉÎÁÔØ ÂÕÄÅÔÅ ÎÅ × ÔÅÍÕ ÒÁÓÓÙÌËÉ, ÎÏ ÓÖÁÌØÔÅÓØ!!!
>
> Ó 12 ÞÁÓÏ× ÌÅÖÉÍÓ =(
>
> ÞÅ ÄÅÌÁÔØ ÕÖÅ ÉÄÅÉ ËÏÎÞÁÌÉÓØ
> ÐÅÒÅËÒÕÔÉÌ sysctl ÕÖÅ ×Ï ×ÓÅ ÓÔÏÒÏÎÙ
> ÓÅÒ×ÁË ÎÁÞÁÌ ÎÅÍÎÏÇÏ ÐÏÌÚÁÔØ, ÎÏ ËÁË ÔÏÌØËÏ
> nginx ÚÁÐÕÓËÁÀ × ËÏÎÓÏÌØ ÓÒÁÚÕ ÎÁÞÉÎÁÅÔ ×ÁÌÉÔÓÑ
>
> Dec š8 23:56:08 mail kernel: interrupt storm detected on "irq257:";
> throttling interrupt source
> Dec š8 23:56:08 mail kernel: Limiting open port RST response from 169 to 50
> packets/sec
> Dec š8 23:56:09 mail kernel: interrupt storm detected on "irq257:";
> throttling interrupt source
> Dec š8 23:56:09 mail kernel: Limiting open port RST response from 230 to 50
> packets/sec
ñ ÐÒÁ×ÉÌØÎÏ ÐÏÎÉÍÁÀ, ÞÔÏ irq257 - ÓÅÔÅ×ÕÈÁ? šåÓÌÉ ÏÎÁ
ÓËÏÌØËÏ-ÎÉÂÕÄØ ÐÒÉÌÉÞÎÁÑ, ÔÏ ÐÏÐÒÏÂÏ×ÁÔØ ÐÏÔÀÎÉÔØ ÂÕÆÅÒÁ/ÚÁÄÅÒÖËÉ
ÐÒÅÒÙ×ÁÎÉÊ. šîÕ ÉÌÉ ÐÒÏÓÔÏ ÐÏÄÎÑÔØ hw.intr_storm_threshold, ÞÔÏÂÙ
ÐÏ ËÒÁÊÎÅÊ ÍÅÒÅ ÓÅÔØ ÐÙÔÁÌÁÓØ ÒÁÂÏÔÁÔØ.
> × ÌÏÇÁÈ ÐÏÌÎÏ:
>
> 80.138.138.94 - - [08/Dec/2010:23:32:16 +0300] "GET / HTTP/1.1" 0 0 "-" "IE
> 7.0"
[...]
> ÞÔÏ ÓÄÅÌÁÎÏ
> 1. fail2ban ÎÁÔÒÁ×ÌÅÎ ÎÁ ÐÏÉÓË ÁÇÅÎÔÁ "IE 7.0" š- ÂÌÏÞÉÔ, ÎÏ ÎÅ ÓÐÁÓÁÅÔ
> 2. ×ÒÅÍÅÎÎÏ location = / { return 200; } - ÎÅ ÓÐÁÓÁÅÔ
> 3. if ($http_user_agent = "IE 7.0" ) { return 412;}
>
> š š š š if ($http_referer = "") { return 412;} - ÔÏÖ ÎÅ ÓÔÁÓÁÅÔ
îÕ ÕÖ ÔÏÇÄÁ return 444;.
> 4. sysctl - ÎÁËÒÕÞÅÎ
>
> sysctl -n kern.ipc.numopensockets
> 90228
reset_timedout_connection on;
èÏÔÑ 90k - ÜÔÏ × ÏÂÝÅÍ ÎÅ ÍÎÏÇÏ.
>
> # netstat -Lan
> Current listen queue sizes (qlen/incqlen/maxqlen)
> Proto Listen š š š š Local Address
> tcp4 š0/0/128 š š š š*.4949
> tcp4 š0/0/4096 š š š 88.212.196.18.443
> tcp4 š0/0/4096 š š š 88.212.196.18.80
> tcp4 š0/0/128 š š š š*.22
> tcp4 š0/0/500 š š š š*.25
> tcp4 š0/0/5 š š š š š88.212.196.18.5666
> tcp4 š0/0/20 š š š š 127.0.0.1.53
> tcp4 š0/0/511 š š š š127.0.0.1.80
> Some tcp sockets may have been created or deleted.
> unix š0/0/1 š š š š š/var/run/fail2ban/fail2ban.sock
> unix š0/0/4 š š š š š/var/run/devd.pipe
îÕ ÐÒÏÓÔÏ ÔÁËÉ ÔÉÛÉÎÁ É ÐÏËÏÊ. š÷ÐÒÏÞÅÍ, ÜÔÏ ×ÅÒÏÑÔÎÏ ÓÌÅÄÓÔ×ÉÅ
"throttling interrupt source".
> # netstat -m
> 1377/36183/37560 mbufs in use (current/cache/total)
> 754/33038/33792/33792 mbuf clusters in use (current/cache/total/max)
á ÇÏ×ÏÒÉÔÅ sysctl ÎÁËÒÕÞÅÎ. šõ ×ÁÓ mbuf cluster'Ï× Ó ÇÕÌØËÉÎ ÆÉÇ.
sysctl kern.ipc.nmbclusters=262144
÷ nginx'Å ÍÏÖÎÏ ÅÝ£ ÐÏÕÍÅÎØÛÁÔØ ÂÕÆÅÒÁ ÎÁ ÓÏËÅÔÁÈ (listen ...
rcvbuf=... sndbuf=...), ÄÁÂÙ ÎÅÍÎÏÇÏ ÐÏÜËÏÎÏÍÉÔØ ÐÁÍÑÔØ ÅÓÌÉ Å£ ÎÅ
È×ÁÔÁÅÔ.
> 36/1628 mbuf+clusters out of packet secondary zone in use (current/cache)
> 0/0/0/16896 4k (page size) jumbo clusters in use (current/cache/total/max)
> 0/0/0/8448 9k jumbo clusters in use (current/cache/total/max)
> 0/0/0/4224 16k jumbo clusters in use (current/cache/total/max)
> 1852K/75121K/76974K bytes allocated to network (current/cache/total)
> 0/17954981/0 requests for mbufs denied (mbufs/clusters/mbuf+clusters)
> 0/0/0 requests for jumbo clusters denied (4k/9k/16k)
> 0/0/0 sfbufs in use (current/peak/max)
> 0 requests for sfbufs denied
> 0 requests for sfbufs delayed
> 1377 requests for I/O initiated by sendfile
> 0 calls to protocol drain routines
>
> ËÁË ÅÝÅ ÂÏÒÏÔØÓÑ ?????
îÕ É vmstat -z ÐÏÓÍÏÔÒÉÔÅ ÄÌÑ ËÏÍÐÌÅËÔÁ, ÔÁÍ ÌÕÞÛÅ ×ÉÄÎÏ ÞÅÇÏ ÎÅ
È×ÁÔÁÅÔ.
îÏ ÐÒÉ ÔÁËÏÍ ËÏÌÉÞÅÓÔ×Å mbuf cluster'Ï× ÐÒÉ ÔÁËÏÍ ËÏÌÉÞÅÓÔ×Å
ÓÏËÅÔÏ× - ÓÔÒÁÎÎÏ ÞÔÏ ÏÎÏ ×ÏÏÂÝÅ ÒÁÂÏÔÁÅÔ, ÄÏÌÖÎÏ ×Ó£ ×
zonelimit'Å ×ÉÓÅÔØ ÂÅÓÐÒÏÂÕÄÎÏ.
Maxim Dounin
