Re: соотношение балансировщи ков/веб-серверов (оффтоп, но...)

[big bond <bondarets@xxxxxxxxx>]

22 октября 2009 г. 23:28 пользователь Igor Sysoev <is@xxxxxxxxxxxxx> написал:
> On Thu, Oct 22, 2009 at 08:20:25PM +0400, big bond wrote:
>
>> CARP только в BSD, а эти замечательные системы у нас в компании не принято
>> использовать. Скорее всего балансер(ы) будут железные, но пока тестируем.
>>
>> Кстати, сегодня пробовали нагрузить конкурентными SSL-сессиями один из
>> тестирумых балансировщиков, использовали программу flood_connect. Я
>> скомпилировал её на линуксовой машине (2хXEON E5420, 2.50GHz, 4GB RAM, SUSE
>> ENT 10.2), выжать смог максимум 16000 соединений, все 4 ядра были загружены
>> на 100%.
>> Коллега скомпилировал на старенькой железке (P3 700MHz, 512MB RAM, FreeBSD
>> 7.2), выжал 7500 соединений и процессор был загружен не более 80%!!!!
>> Сам балансировщик при этом тоже неплохо был загружен.
>> Как такое возможно? Старая железка отстала всего в два раза от современного
>> неслабого сервера!
>> Запускали так: *flood_connect -S -f 10 -p 443 адрес_балансировщика*
>> -f - количество форков
>> -S - SSL-режим
>
> Скорее всего, разные шифры.
> Запустите "openssl s_client -connect host:port" и посмотрите Cipher.
>
Посмотрели, Cipher одинаковый...

Может кто подскажет какие-то конкретные значения sysctl или другие
настройки, чтобы порешать проблему?
somaxconn пробовал увеличить с 128 до 1024 - не помогло
вот мой ulimit:
 ulimit -a
core file size          (blocks, -c) 0
data seg size           (kbytes, -d) unlimited
file size               (blocks, -f) unlimited
pending signals                 (-i) 38911
max locked memory       (kbytes, -l) 32
max memory size         (kbytes, -m) unlimited
open files                      (-n) 15000
pipe size            (512 bytes, -p) 8
POSIX message queues     (bytes, -q) 819200
stack size              (kbytes, -s) 8192
cpu time               (seconds, -t) unlimited
max user processes              (-u) 38911
virtual memory          (kbytes, -v) unlimited
file locks                      (-x) unlimited

вот sysctl.conf, был "из коробки":
cat /etc/sysctl.conf | grep -v "#"
net.ipv4.icmp_echo_ignore_broadcasts = 1
net.ipv4.conf.all.rp_filter = 1

sysctl -a во вложении.

uname -a
Linux logger 2.6.16.21-0.8-bigsmp #1 SMP Mon Jul 3 18:25:39 UTC 2006
i686 i686 i386 GNU/Linux

>
> --
> Игорь Сысоев
> http://sysoev.ru
>
>

dev.scsi.logging_level = 0
dev.raid.speed_limit_max = 200000
dev.raid.speed_limit_min = 1000
dev.hpet.max-user-freq = 64
dev.rtc.max-user-freq = 64
net.unix.max_dgram_qlen = 10
net.token-ring.rif_timeout = 150000
net.ipv4.conf.tun0.promote_secondaries = 0
net.ipv4.conf.tun0.force_igmp_version = 0
net.ipv4.conf.tun0.disable_policy = 0
net.ipv4.conf.tun0.disable_xfrm = 0
net.ipv4.conf.tun0.arp_ignore = 0
net.ipv4.conf.tun0.arp_announce = 0
net.ipv4.conf.tun0.arp_filter = 0
net.ipv4.conf.tun0.tag = 0
net.ipv4.conf.tun0.log_martians = 0
net.ipv4.conf.tun0.bootp_relay = 0
net.ipv4.conf.tun0.medium_id = 0
net.ipv4.conf.tun0.proxy_arp = 0
net.ipv4.conf.tun0.accept_source_route = 1
net.ipv4.conf.tun0.send_redirects = 1
net.ipv4.conf.tun0.rp_filter = 0
net.ipv4.conf.tun0.shared_media = 1
net.ipv4.conf.tun0.secure_redirects = 1
net.ipv4.conf.tun0.accept_redirects = 1
net.ipv4.conf.tun0.mc_forwarding = 0
net.ipv4.conf.tun0.forwarding = 0
net.ipv4.conf.eth0.promote_secondaries = 0
net.ipv4.conf.eth0.force_igmp_version = 0
net.ipv4.conf.eth0.disable_policy = 0
net.ipv4.conf.eth0.disable_xfrm = 0
net.ipv4.conf.eth0.arp_ignore = 0
net.ipv4.conf.eth0.arp_announce = 0
net.ipv4.conf.eth0.arp_filter = 0
net.ipv4.conf.eth0.tag = 0
net.ipv4.conf.eth0.log_martians = 0
net.ipv4.conf.eth0.bootp_relay = 0
net.ipv4.conf.eth0.medium_id = 0
net.ipv4.conf.eth0.proxy_arp = 0
net.ipv4.conf.eth0.accept_source_route = 1
net.ipv4.conf.eth0.send_redirects = 1
net.ipv4.conf.eth0.rp_filter = 0
net.ipv4.conf.eth0.shared_media = 1
net.ipv4.conf.eth0.secure_redirects = 1
net.ipv4.conf.eth0.accept_redirects = 1
net.ipv4.conf.eth0.mc_forwarding = 0
net.ipv4.conf.eth0.forwarding = 0
net.ipv4.conf.eth1.promote_secondaries = 0
net.ipv4.conf.eth1.force_igmp_version = 0
net.ipv4.conf.eth1.disable_policy = 0
net.ipv4.conf.eth1.disable_xfrm = 0
net.ipv4.conf.eth1.arp_ignore = 0
net.ipv4.conf.eth1.arp_announce = 0
net.ipv4.conf.eth1.arp_filter = 0
net.ipv4.conf.eth1.tag = 0
net.ipv4.conf.eth1.log_martians = 0
net.ipv4.conf.eth1.bootp_relay = 0
net.ipv4.conf.eth1.medium_id = 0
net.ipv4.conf.eth1.proxy_arp = 0
net.ipv4.conf.eth1.accept_source_route = 1
net.ipv4.conf.eth1.send_redirects = 1
net.ipv4.conf.eth1.rp_filter = 0
net.ipv4.conf.eth1.shared_media = 1
net.ipv4.conf.eth1.secure_redirects = 1
net.ipv4.conf.eth1.accept_redirects = 1
net.ipv4.conf.eth1.mc_forwarding = 0
net.ipv4.conf.eth1.forwarding = 0
net.ipv4.conf.default.promote_secondaries = 0
net.ipv4.conf.default.force_igmp_version = 0
net.ipv4.conf.default.disable_policy = 0
net.ipv4.conf.default.disable_xfrm = 0
net.ipv4.conf.default.arp_ignore = 0
net.ipv4.conf.default.arp_announce = 0
net.ipv4.conf.default.arp_filter = 0
net.ipv4.conf.default.tag = 0
net.ipv4.conf.default.log_martians = 0
net.ipv4.conf.default.bootp_relay = 0
net.ipv4.conf.default.medium_id = 0
net.ipv4.conf.default.proxy_arp = 0
net.ipv4.conf.default.accept_source_route = 1
net.ipv4.conf.default.send_redirects = 1
net.ipv4.conf.default.rp_filter = 0
net.ipv4.conf.default.shared_media = 1
net.ipv4.conf.default.secure_redirects = 1
net.ipv4.conf.default.accept_redirects = 1
net.ipv4.conf.default.mc_forwarding = 0
net.ipv4.conf.default.forwarding = 0
net.ipv4.conf.all.promote_secondaries = 0
net.ipv4.conf.all.force_igmp_version = 0
net.ipv4.conf.all.disable_policy = 0
net.ipv4.conf.all.disable_xfrm = 0
net.ipv4.conf.all.arp_ignore = 0
net.ipv4.conf.all.arp_announce = 0
net.ipv4.conf.all.arp_filter = 0
net.ipv4.conf.all.tag = 0
net.ipv4.conf.all.log_martians = 0
net.ipv4.conf.all.bootp_relay = 0
net.ipv4.conf.all.medium_id = 0
net.ipv4.conf.all.proxy_arp = 0
net.ipv4.conf.all.accept_source_route = 0
net.ipv4.conf.all.send_redirects = 1
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.all.shared_media = 1
net.ipv4.conf.all.secure_redirects = 1
net.ipv4.conf.all.accept_redirects = 1
net.ipv4.conf.all.mc_forwarding = 0
net.ipv4.conf.all.forwarding = 0
net.ipv4.conf.lo.promote_secondaries = 0
net.ipv4.conf.lo.force_igmp_version = 0
net.ipv4.conf.lo.disable_policy = 1
net.ipv4.conf.lo.disable_xfrm = 1
net.ipv4.conf.lo.arp_ignore = 0
net.ipv4.conf.lo.arp_announce = 0
net.ipv4.conf.lo.arp_filter = 0
net.ipv4.conf.lo.tag = 0
net.ipv4.conf.lo.log_martians = 0
net.ipv4.conf.lo.bootp_relay = 0
net.ipv4.conf.lo.medium_id = 0
net.ipv4.conf.lo.proxy_arp = 0
net.ipv4.conf.lo.accept_source_route = 1
net.ipv4.conf.lo.send_redirects = 1
net.ipv4.conf.lo.rp_filter = 0
net.ipv4.conf.lo.shared_media = 1
net.ipv4.conf.lo.secure_redirects = 1
net.ipv4.conf.lo.accept_redirects = 1
net.ipv4.conf.lo.mc_forwarding = 0
net.ipv4.conf.lo.forwarding = 0
net.ipv4.neigh.tun0.base_reachable_time_ms = 30000
net.ipv4.neigh.tun0.retrans_time_ms = 1000
net.ipv4.neigh.tun0.locktime = 100
net.ipv4.neigh.tun0.proxy_delay = 80
net.ipv4.neigh.tun0.anycast_delay = 100
net.ipv4.neigh.tun0.proxy_qlen = 64
net.ipv4.neigh.tun0.unres_qlen = 3
net.ipv4.neigh.tun0.gc_stale_time = 60
net.ipv4.neigh.tun0.delay_first_probe_time = 5
net.ipv4.neigh.tun0.base_reachable_time = 30
net.ipv4.neigh.tun0.retrans_time = 100
net.ipv4.neigh.tun0.app_solicit = 0
net.ipv4.neigh.tun0.ucast_solicit = 3
net.ipv4.neigh.tun0.mcast_solicit = 3
net.ipv4.neigh.eth0.base_reachable_time_ms = 30000
net.ipv4.neigh.eth0.retrans_time_ms = 1000
net.ipv4.neigh.eth0.locktime = 100
net.ipv4.neigh.eth0.proxy_delay = 80
net.ipv4.neigh.eth0.anycast_delay = 100
net.ipv4.neigh.eth0.proxy_qlen = 64
net.ipv4.neigh.eth0.unres_qlen = 3
net.ipv4.neigh.eth0.gc_stale_time = 60
net.ipv4.neigh.eth0.delay_first_probe_time = 5
net.ipv4.neigh.eth0.base_reachable_time = 30
net.ipv4.neigh.eth0.retrans_time = 100
net.ipv4.neigh.eth0.app_solicit = 0
net.ipv4.neigh.eth0.ucast_solicit = 3
net.ipv4.neigh.eth0.mcast_solicit = 3
net.ipv4.neigh.eth1.base_reachable_time_ms = 30000
net.ipv4.neigh.eth1.retrans_time_ms = 1000
net.ipv4.neigh.eth1.locktime = 100
net.ipv4.neigh.eth1.proxy_delay = 80
net.ipv4.neigh.eth1.anycast_delay = 100
net.ipv4.neigh.eth1.proxy_qlen = 64
net.ipv4.neigh.eth1.unres_qlen = 3
net.ipv4.neigh.eth1.gc_stale_time = 60
net.ipv4.neigh.eth1.delay_first_probe_time = 5
net.ipv4.neigh.eth1.base_reachable_time = 30
net.ipv4.neigh.eth1.retrans_time = 100
net.ipv4.neigh.eth1.app_solicit = 0
net.ipv4.neigh.eth1.ucast_solicit = 3
net.ipv4.neigh.eth1.mcast_solicit = 3
net.ipv4.neigh.lo.base_reachable_time_ms = 30000
net.ipv4.neigh.lo.retrans_time_ms = 1000
net.ipv4.neigh.lo.locktime = 100
net.ipv4.neigh.lo.proxy_delay = 80
net.ipv4.neigh.lo.anycast_delay = 100
net.ipv4.neigh.lo.proxy_qlen = 64
net.ipv4.neigh.lo.unres_qlen = 3
net.ipv4.neigh.lo.gc_stale_time = 60
net.ipv4.neigh.lo.delay_first_probe_time = 5
net.ipv4.neigh.lo.base_reachable_time = 30
net.ipv4.neigh.lo.retrans_time = 100
net.ipv4.neigh.lo.app_solicit = 0
net.ipv4.neigh.lo.ucast_solicit = 3
net.ipv4.neigh.lo.mcast_solicit = 3
net.ipv4.neigh.default.base_reachable_time_ms = 30000
net.ipv4.neigh.default.retrans_time_ms = 1000
net.ipv4.neigh.default.gc_thresh3 = 1024
net.ipv4.neigh.default.gc_thresh2 = 512
net.ipv4.neigh.default.gc_thresh1 = 128
net.ipv4.neigh.default.gc_interval = 30
net.ipv4.neigh.default.locktime = 100
net.ipv4.neigh.default.proxy_delay = 80
net.ipv4.neigh.default.anycast_delay = 100
net.ipv4.neigh.default.proxy_qlen = 64
net.ipv4.neigh.default.unres_qlen = 3
net.ipv4.neigh.default.gc_stale_time = 60
net.ipv4.neigh.default.delay_first_probe_time = 5
net.ipv4.neigh.default.base_reachable_time = 30
net.ipv4.neigh.default.retrans_time = 100
net.ipv4.neigh.default.app_solicit = 0
net.ipv4.neigh.default.ucast_solicit = 3
net.ipv4.neigh.default.mcast_solicit = 3
net.ipv4.tcp_dma_copybreak = 4096
net.ipv4.tcp_abc = 1
net.ipv4.tcp_congestion_control = reno
net.ipv4.tcp_tso_win_divisor = 3
net.ipv4.tcp_moderate_rcvbuf = 1
net.ipv4.tcp_no_metrics_save = 0
net.ipv4.ipfrag_max_dist = 64
net.ipv4.ipfrag_secret_interval = 600
net.ipv4.tcp_low_latency = 0
net.ipv4.tcp_frto = 0
net.ipv4.tcp_tw_reuse = 0
net.ipv4.icmp_ratemask = 6168
net.ipv4.icmp_ratelimit = 250
net.ipv4.tcp_adv_win_scale = 2
net.ipv4.tcp_app_win = 31
net.ipv4.tcp_rmem = 4096        87380   174760
net.ipv4.tcp_wmem = 4096        16384   131072
net.ipv4.tcp_mem = 98304        131072  196608
net.ipv4.tcp_dsack = 1
net.ipv4.tcp_ecn = 0
net.ipv4.tcp_reordering = 3
net.ipv4.tcp_fack = 1
net.ipv4.tcp_orphan_retries = 0
net.ipv4.inet_peer_gc_maxtime = 120
net.ipv4.inet_peer_gc_mintime = 10
net.ipv4.inet_peer_maxttl = 600
net.ipv4.inet_peer_minttl = 120
net.ipv4.inet_peer_threshold = 65664
net.ipv4.igmp_max_msf = 10
net.ipv4.igmp_max_memberships = 20
net.ipv4.route.secret_interval = 600
net.ipv4.route.min_adv_mss = 256
net.ipv4.route.min_pmtu = 552
net.ipv4.route.mtu_expires = 600
net.ipv4.route.gc_elasticity = 8
net.ipv4.route.error_burst = 1250
net.ipv4.route.error_cost = 250
net.ipv4.route.redirect_silence = 5120
net.ipv4.route.redirect_number = 9
net.ipv4.route.redirect_load = 5
net.ipv4.route.gc_interval = 60
net.ipv4.route.gc_timeout = 300
net.ipv4.route.gc_min_interval_ms = 500
net.ipv4.route.gc_min_interval = 0
net.ipv4.route.max_size = 4194304
net.ipv4.route.gc_thresh = 262144
net.ipv4.route.max_delay = 10
net.ipv4.route.min_delay = 2
net.ipv4.icmp_errors_use_inbound_ifaddr = 0
net.ipv4.icmp_ignore_bogus_error_responses = 1
net.ipv4.icmp_echo_ignore_broadcasts = 1
net.ipv4.icmp_echo_ignore_all = 0
net.ipv4.ip_local_port_range = 32768    61000
net.ipv4.tcp_max_syn_backlog = 1024
net.ipv4.tcp_rfc1337 = 0
net.ipv4.tcp_stdurg = 0
net.ipv4.tcp_abort_on_overflow = 0
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_fin_timeout = 60
net.ipv4.tcp_retries2 = 15
net.ipv4.tcp_retries1 = 3
net.ipv4.tcp_keepalive_intvl = 75
net.ipv4.tcp_keepalive_probes = 9
net.ipv4.tcp_keepalive_time = 7200
net.ipv4.ipfrag_time = 30
net.ipv4.ip_dynaddr = 0
net.ipv4.ipfrag_low_thresh = 196608
net.ipv4.ipfrag_high_thresh = 262144
net.ipv4.tcp_max_tw_buckets = 180000
net.ipv4.tcp_max_orphans = 32768
net.ipv4.tcp_synack_retries = 5
net.ipv4.tcp_syn_retries = 5
net.ipv4.ip_nonlocal_bind = 0
net.ipv4.ip_no_pmtu_disc = 0
net.ipv4.ip_autoconfig = 0
net.ipv4.ip_default_ttl = 64
net.ipv4.ip_forward = 0
net.ipv4.tcp_retrans_collapse = 1
net.ipv4.tcp_sack = 1
net.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_timestamps = 1
net.core.netdev_budget = 300
net.core.somaxconn = 1024
net.core.optmem_max = 10240
net.core.message_burst = 10
net.core.message_cost = 5
net.core.netdev_max_backlog = 1000
net.core.dev_weight = 64
net.core.rmem_default = 108544
net.core.wmem_default = 108544
net.core.rmem_max = 131071
net.core.wmem_max = 131071
vm.swap_token_timeout = 300
vm.legacy_va_layout = 0
vm.vfs_cache_pressure = 100
vm.block_dump = 0
vm.laptop_mode = 0
vm.max_map_count = 65536
vm.percpu_pagelist_fraction = 0
vm.min_free_kbytes = 3831
vm.drop_caches = 0
vm.lowmem_reserve_ratio = 256   256     32
vm.hugetlb_shm_group = 0
vm.nr_hugepages = 0
vm.swappiness = 60
vm.nr_pdflush_threads = 2
vm.dirty_expire_centisecs = 3000
vm.dirty_writeback_centisecs = 500
vm.dirty_ratio = 29
vm.dirty_background_ratio = 7
vm.page-cluster = 3
vm.overcommit_ratio = 50
vm.overcommit_memory = 0
kernel.affinity_load_balancing = 0
kernel.rcu_mask = 0
kernel.HZ = 250
kernel.acpi_video_flags = 0
kernel.suid_dumpable = 0
kernel.randomize_va_space = 1
kernel.bootloader_type = 113
kernel.unknown_nmi_panic = 0
kernel.ngroups_max = 65536
kernel.printk_ratelimit_burst = 10
kernel.printk_ratelimit = 5
kernel.panic_on_oops = 0
kernel.pid_max = 32768
kernel.overflowgid = 65534
kernel.overflowuid = 65534
kernel.pty.nr = 6
kernel.pty.max = 4096
kernel.random.uuid = 168e2294-6a34-4e59-95d9-799a39cf16bf
kernel.random.boot_id = 13524a56-12c9-4442-9541-e9813f1a54fe
kernel.random.write_wakeup_threshold = 128
kernel.random.read_wakeup_threshold = 64
kernel.random.entropy_avail = 169
kernel.random.poolsize = 4096
kernel.threads-max = 77823
kernel.cad_pid = 1
kernel.sysrq = 0
kernel.sem = 250        256000  32      1024
kernel.msgmnb = 65536
kernel.msgmni = 16
kernel.msgmax = 65536
kernel.shmmni = 4096
kernel.shmall = 268435200
kernel.shmmax = 4294967295
kernel.acct = 4 2       30
kernel.hotplug = 
kernel.modprobe = /sbin/modprobe
kernel.printk = 1       4       1       7
kernel.ctrl-alt-del = 0
kernel.real-root-dev = 256
kernel.cap-bound = -257
kernel.unsupported = 1
kernel.tainted = 0
kernel.core_pattern = core
kernel.core_uses_pid = 0
kernel.panic = 0
kernel.domainname = 
kernel.hostname = logger
kernel.version = #1 SMP Mon Jul 3 18:25:39 UTC 2006
kernel.osrelease = 2.6.16.21-0.8-bigsmp
kernel.ostype = Linux
fs.mqueue.msgsize_max = 8192
fs.mqueue.msg_max = 10
fs.mqueue.queues_max = 256
fs.quota.warnings = 1
fs.quota.syncs = 18
fs.quota.free_dquots = 0
fs.quota.allocated_dquots = 0
fs.quota.cache_hits = 0
fs.quota.writes = 0
fs.quota.reads = 0
fs.quota.drops = 0
fs.quota.lookups = 0
fs.inotify.max_queued_events = 16384
fs.inotify.max_user_watches = 8192
fs.inotify.max_user_instances = 128
fs.aio-max-nr = 65536
fs.aio-nr = 0
fs.lease-break-time = 45
fs.dir-notify-enable = 1
fs.leases-enable = 1
fs.overflowgid = 65534
fs.overflowuid = 65534
fs.dentry-state = 48195 45601   45      0       0       0
fs.file-max = 372163
fs.file-nr = 1280       0       372163
fs.inode-state = 42917  973     0       0       0       0       0
fs.inode-nr = 42917     973