求助！线上服务的nginx在对客户端提供服务的时候，会有千分之一的情况给 Connection Reset

pcman.zhang · 2013-07-19T18:20:56+00:00

Hello! 2013/7/19 杨进: > ngx-memc模块与srcache-nginx-module模块使用到memcached软&#x4EF6...

求助！线上服务的nginx在对客户端提供服务的时候，会有千分之一的情况给 Connection Reset

pcman.zhang

1. 机器配置
    64GB 内存
    8核 16线程的志强处理器
    运行CentOS 6.3

2. nginx 承担的工作
    主要是提供反向代理工作
    其中有proxy_cache，量大概在600GB
    HIT命中率在60%左右
    平常提供QPS 200上下。

运行nginx版本：1.2.0

附：
. 内核参数

net.netfilter.nf_log.0 = NONE
net.netfilter.nf_log.1 = NONE
net.netfilter.nf_log.2 = NONE
net.netfilter.nf_log.3 = NONE
net.netfilter.nf_log.4 = NONE
net.netfilter.nf_log.5 = NONE
net.netfilter.nf_log.6 = NONE
net.netfilter.nf_log.7 = NONE
net.netfilter.nf_log.8 = NONE
net.netfilter.nf_log.9 = NONE
net.netfilter.nf_log.10 = NONE
net.netfilter.nf_log.11 = NONE
net.netfilter.nf_log.12 = NONE
net.core.somaxconn = 128
net.core.xfrm_aevent_etime = 10
net.core.xfrm_aevent_rseqth = 2
net.core.xfrm_larval_drop = 1
net.core.xfrm_acq_expires = 30
net.core.wmem_max = 1048576
net.core.rmem_max = 131071
net.core.wmem_default = 229376
net.core.rmem_default = 229376
net.core.dev_weight = 64
net.core.netdev_max_backlog = 1000
net.core.message_cost = 5
net.core.message_burst = 10
net.core.optmem_max = 20480
net.core.rps_sock_flow_entries = 0
net.core.netdev_budget = 300
net.core.warnings = 1
net.ipv4.route.gc_thresh = 524288
net.ipv4.route.max_size = 8388608
net.ipv4.route.gc_min_interval = 0
net.ipv4.route.gc_min_interval_ms = 500
net.ipv4.route.gc_timeout = 300
net.ipv4.route.gc_interval = 60
net.ipv4.route.redirect_load = 20
net.ipv4.route.redirect_number = 9
net.ipv4.route.redirect_silence = 20480
net.ipv4.route.error_cost = 1000
net.ipv4.route.error_burst = 5000
net.ipv4.route.gc_elasticity = 8
net.ipv4.route.mtu_expires = 600
net.ipv4.route.min_pmtu = 552
net.ipv4.route.min_adv_mss = 256
net.ipv4.route.secret_interval = 600
net.ipv4.neigh.default.mcast_solicit = 3
net.ipv4.neigh.default.ucast_solicit = 3
net.ipv4.neigh.default.app_solicit = 0
net.ipv4.neigh.default.retrans_time = 99
net.ipv4.neigh.default.base_reachable_time = 30
net.ipv4.neigh.default.delay_first_probe_time = 5
net.ipv4.neigh.default.gc_stale_time = 60
net.ipv4.neigh.default.unres_qlen = 3
net.ipv4.neigh.default.proxy_qlen = 64
net.ipv4.neigh.default.anycast_delay = 99
net.ipv4.neigh.default.proxy_delay = 79
net.ipv4.neigh.default.locktime = 99
net.ipv4.neigh.default.retrans_time_ms = 1000
net.ipv4.neigh.default.base_reachable_time_ms = 30000
net.ipv4.neigh.default.gc_interval = 30
net.ipv4.neigh.default.gc_thresh1 = 128
net.ipv4.neigh.default.gc_thresh2 = 512
net.ipv4.neigh.default.gc_thresh3 = 1024
net.ipv4.neigh.lo.mcast_solicit = 3
net.ipv4.neigh.lo.ucast_solicit = 3
net.ipv4.neigh.lo.app_solicit = 0
net.ipv4.neigh.lo.retrans_time = 99
net.ipv4.neigh.lo.base_reachable_time = 30
net.ipv4.neigh.lo.delay_first_probe_time = 5
net.ipv4.neigh.lo.gc_stale_time = 60
net.ipv4.neigh.lo.unres_qlen = 3
net.ipv4.neigh.lo.proxy_qlen = 64
net.ipv4.neigh.lo.anycast_delay = 99
net.ipv4.neigh.lo.proxy_delay = 79
net.ipv4.neigh.lo.locktime = 99
net.ipv4.neigh.lo.retrans_time_ms = 1000
net.ipv4.neigh.lo.base_reachable_time_ms = 30000
net.ipv4.neigh.em4.mcast_solicit = 3
net.ipv4.neigh.em4.ucast_solicit = 3
net.ipv4.neigh.em4.app_solicit = 0
net.ipv4.neigh.em4.retrans_time = 99
net.ipv4.neigh.em4.base_reachable_time = 30
net.ipv4.neigh.em4.delay_first_probe_time = 5
net.ipv4.neigh.em4.gc_stale_time = 60
net.ipv4.neigh.em4.unres_qlen = 3
net.ipv4.neigh.em4.proxy_qlen = 64
net.ipv4.neigh.em4.anycast_delay = 99
net.ipv4.neigh.em4.proxy_delay = 79
net.ipv4.neigh.em4.locktime = 99
net.ipv4.neigh.em4.retrans_time_ms = 1000
net.ipv4.neigh.em4.base_reachable_time_ms = 30000
net.ipv4.neigh.em2.mcast_solicit = 3
net.ipv4.neigh.em2.ucast_solicit = 3
net.ipv4.neigh.em2.app_solicit = 0
net.ipv4.neigh.em2.retrans_time = 99
net.ipv4.neigh.em2.base_reachable_time = 30
net.ipv4.neigh.em2.delay_first_probe_time = 5
net.ipv4.neigh.em2.gc_stale_time = 60
net.ipv4.neigh.em2.unres_qlen = 3
net.ipv4.neigh.em2.proxy_qlen = 64
net.ipv4.neigh.em2.anycast_delay = 99
net.ipv4.neigh.em2.proxy_delay = 79
net.ipv4.neigh.em2.locktime = 99
net.ipv4.neigh.em2.retrans_time_ms = 1000
net.ipv4.neigh.em2.base_reachable_time_ms = 30000
net.ipv4.neigh.em3.mcast_solicit = 3
net.ipv4.neigh.em3.ucast_solicit = 3
net.ipv4.neigh.em3.app_solicit = 0
net.ipv4.neigh.em3.retrans_time = 99
net.ipv4.neigh.em3.base_reachable_time = 30
net.ipv4.neigh.em3.delay_first_probe_time = 5
net.ipv4.neigh.em3.gc_stale_time = 60
net.ipv4.neigh.em3.unres_qlen = 3
net.ipv4.neigh.em3.proxy_qlen = 64
net.ipv4.neigh.em3.anycast_delay = 99
net.ipv4.neigh.em3.proxy_delay = 79
net.ipv4.neigh.em3.locktime = 99
net.ipv4.neigh.em3.retrans_time_ms = 1000
net.ipv4.neigh.em3.base_reachable_time_ms = 30000
net.ipv4.neigh.em1.mcast_solicit = 3
net.ipv4.neigh.em1.ucast_solicit = 3
net.ipv4.neigh.em1.app_solicit = 0
net.ipv4.neigh.em1.retrans_time = 99
net.ipv4.neigh.em1.base_reachable_time = 30
net.ipv4.neigh.em1.delay_first_probe_time = 5
net.ipv4.neigh.em1.gc_stale_time = 60
net.ipv4.neigh.em1.unres_qlen = 3
net.ipv4.neigh.em1.proxy_qlen = 64
net.ipv4.neigh.em1.anycast_delay = 99
net.ipv4.neigh.em1.proxy_delay = 79
net.ipv4.neigh.em1.locktime = 99
net.ipv4.neigh.em1.retrans_time_ms = 1000
net.ipv4.neigh.em1.base_reachable_time_ms = 30000
net.ipv4.tcp_timestamps = 1
net.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_sack = 1
net.ipv4.tcp_retrans_collapse = 1
net.ipv4.ip_default_ttl = 64
net.ipv4.ip_no_pmtu_disc = 0
net.ipv4.ip_nonlocal_bind = 1
net.ipv4.tcp_syn_retries = 5
net.ipv4.tcp_synack_retries = 3
net.ipv4.tcp_max_orphans = 3276800
net.ipv4.tcp_max_tw_buckets = 5000
net.ipv4.ip_dynaddr = 0
net.ipv4.tcp_keepalive_time = 300
net.ipv4.tcp_keepalive_probes = 9
net.ipv4.tcp_keepalive_intvl = 75
net.ipv4.tcp_retries1 = 3
net.ipv4.tcp_retries2 = 15
net.ipv4.tcp_fin_timeout = 15
net.ipv4.tcp_syncookies = 0
net.ipv4.tcp_tw_recycle = 0
net.ipv4.tcp_abort_on_overflow = 0
net.ipv4.tcp_stdurg = 0
net.ipv4.tcp_rfc1337 = 0
net.ipv4.tcp_max_syn_backlog = 65536
net.ipv4.ip_local_port_range = 5000    65000
net.ipv4.ip_local_reserved_ports =
net.ipv4.igmp_max_memberships = 20
net.ipv4.igmp_max_msf = 10
net.ipv4.inet_peer_threshold = 65664
net.ipv4.inet_peer_minttl = 120
net.ipv4.inet_peer_maxttl = 600
net.ipv4.inet_peer_gc_mintime = 10
net.ipv4.inet_peer_gc_maxtime = 120
net.ipv4.tcp_orphan_retries = 0
net.ipv4.tcp_fack = 1
net.ipv4.tcp_reordering = 3
net.ipv4.tcp_ecn = 2
net.ipv4.tcp_dsack = 1
net.ipv4.tcp_mem = 94500000    915000000    927000000
net.ipv4.tcp_wmem = 4096    16384    4194304
net.ipv4.tcp_rmem = 4096    87380    4194304
net.ipv4.tcp_app_win = 31
net.ipv4.tcp_adv_win_scale = 2
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_frto = 2
net.ipv4.tcp_frto_response = 0
net.ipv4.tcp_low_latency = 0
net.ipv4.tcp_no_metrics_save = 0
net.ipv4.tcp_moderate_rcvbuf = 1
net.ipv4.tcp_tso_win_divisor = 3
net.ipv4.tcp_congestion_control = cubic
net.ipv4.tcp_abc = 0
net.ipv4.tcp_mtu_probing = 0
net.ipv4.tcp_base_mss = 512
net.ipv4.tcp_workaround_signed_windows = 0
net.ipv4.tcp_dma_copybreak = 4096
net.ipv4.tcp_slow_start_after_idle = 1
net.ipv4.cipso_cache_enable = 1
net.ipv4.cipso_cache_bucket_size = 10
net.ipv4.cipso_rbm_optfmt = 0
net.ipv4.cipso_rbm_strictvalid = 1
net.ipv4.tcp_available_congestion_control = cubic reno
net.ipv4.tcp_allowed_congestion_control = cubic reno
net.ipv4.tcp_max_ssthresh = 0
net.ipv4.tcp_thin_linear_timeouts = 0
net.ipv4.tcp_thin_dupack = 0
net.ipv4.udp_mem = 6181536    8242048    12363072
net.ipv4.udp_rmem_min = 4096
net.ipv4.udp_wmem_min = 4096
net.ipv4.conf.all.forwarding = 1
net.ipv4.conf.all.mc_forwarding = 0
net.ipv4.conf.all.accept_redirects = 0
net.ipv4.conf.all.secure_redirects = 1
net.ipv4.conf.all.shared_media = 1
net.ipv4.conf.all.rp_filter = 2
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.all.accept_source_route = 0
net.ipv4.conf.all.src_valid_mark = 0
net.ipv4.conf.all.proxy_arp = 0
net.ipv4.conf.all.medium_id = 0
net.ipv4.conf.all.bootp_relay = 0
net.ipv4.conf.all.log_martians = 0
net.ipv4.conf.all.tag = 0
net.ipv4.conf.all.arp_filter = 0
net.ipv4.conf.all.arp_announce = 2
net.ipv4.conf.all.arp_ignore = 2
net.ipv4.conf.all.arp_accept = 0
net.ipv4.conf.all.arp_notify = 0
net.ipv4.conf.all.proxy_arp_pvlan = 0
net.ipv4.conf.all.disable_xfrm = 0
net.ipv4.conf.all.disable_policy = 0
net.ipv4.conf.all.force_igmp_version = 0
net.ipv4.conf.all.promote_secondaries = 0
net.ipv4.conf.all.accept_local = 0
net.ipv4.conf.default.forwarding = 1
net.ipv4.conf.default.mc_forwarding = 0
net.ipv4.conf.default.accept_redirects = 1
net.ipv4.conf.default.secure_redirects = 1
net.ipv4.conf.default.shared_media = 1
net.ipv4.conf.default.rp_filter = 2
net.ipv4.conf.default.send_redirects = 1
net.ipv4.conf.default.accept_source_route = 0
net.ipv4.conf.default.src_valid_mark = 0
net.ipv4.conf.default.proxy_arp = 0
net.ipv4.conf.default.medium_id = 0
net.ipv4.conf.default.bootp_relay = 0
net.ipv4.conf.default.log_martians = 0
net.ipv4.conf.default.tag = 0
net.ipv4.conf.default.arp_filter = 0
net.ipv4.conf.default.arp_announce = 0
net.ipv4.conf.default.arp_ignore = 0
net.ipv4.conf.default.arp_accept = 0
net.ipv4.conf.default.arp_notify = 0
net.ipv4.conf.default.proxy_arp_pvlan = 0
net.ipv4.conf.default.disable_xfrm = 0
net.ipv4.conf.default.disable_policy = 0
net.ipv4.conf.default.force_igmp_version = 0
net.ipv4.conf.default.promote_secondaries = 0
net.ipv4.conf.default.accept_local = 0
net.ipv4.conf.lo.forwarding = 1
net.ipv4.conf.lo.mc_forwarding = 0
net.ipv4.conf.lo.accept_redirects = 1
net.ipv4.conf.lo.secure_redirects = 1
net.ipv4.conf.lo.shared_media = 1
net.ipv4.conf.lo.rp_filter = 0
net.ipv4.conf.lo.send_redirects = 1
net.ipv4.conf.lo.accept_source_route = 0
net.ipv4.conf.lo.src_valid_mark = 0
net.ipv4.conf.lo.proxy_arp = 0
net.ipv4.conf.lo.medium_id = 0
net.ipv4.conf.lo.bootp_relay = 0
net.ipv4.conf.lo.log_martians = 0
net.ipv4.conf.lo.tag = 0
net.ipv4.conf.lo.arp_filter = 0
net.ipv4.conf.lo.arp_announce = 0
net.ipv4.conf.lo.arp_ignore = 0
net.ipv4.conf.lo.arp_accept = 0
net.ipv4.conf.lo.arp_notify = 0
net.ipv4.conf.lo.proxy_arp_pvlan = 0
net.ipv4.conf.lo.disable_xfrm = 1
net.ipv4.conf.lo.disable_policy = 1
net.ipv4.conf.lo.force_igmp_version = 0
net.ipv4.conf.lo.promote_secondaries = 0
net.ipv4.conf.lo.accept_local = 0
net.ipv4.conf.em4.forwarding = 1
net.ipv4.conf.em4.mc_forwarding = 0
net.ipv4.conf.em4.accept_redirects = 1
net.ipv4.conf.em4.secure_redirects = 1
net.ipv4.conf.em4.shared_media = 1
net.ipv4.conf.em4.rp_filter = 2
net.ipv4.conf.em4.send_redirects = 1
net.ipv4.conf.em4.accept_source_route = 0
net.ipv4.conf.em4.src_valid_mark = 0
net.ipv4.conf.em4.proxy_arp = 0
net.ipv4.conf.em4.medium_id = 0
net.ipv4.conf.em4.bootp_relay = 0
net.ipv4.conf.em4.log_martians = 0
net.ipv4.conf.em4.tag = 0
net.ipv4.conf.em4.arp_filter = 0
net.ipv4.conf.em4.arp_announce = 0
net.ipv4.conf.em4.arp_ignore = 0
net.ipv4.conf.em4.arp_accept = 0
net.ipv4.conf.em4.arp_notify = 0
net.ipv4.conf.em4.proxy_arp_pvlan = 0
net.ipv4.conf.em4.disable_xfrm = 0
net.ipv4.conf.em4.disable_policy = 0
net.ipv4.conf.em4.force_igmp_version = 0
net.ipv4.conf.em4.promote_secondaries = 0
net.ipv4.conf.em4.accept_local = 0
net.ipv4.conf.em2.forwarding = 1
net.ipv4.conf.em2.mc_forwarding = 0
net.ipv4.conf.em2.accept_redirects = 1
net.ipv4.conf.em2.secure_redirects = 1
net.ipv4.conf.em2.shared_media = 1
net.ipv4.conf.em2.rp_filter = 2
net.ipv4.conf.em2.send_redirects = 1
net.ipv4.conf.em2.accept_source_route = 0
net.ipv4.conf.em2.src_valid_mark = 0
net.ipv4.conf.em2.proxy_arp = 0
net.ipv4.conf.em2.medium_id = 0
net.ipv4.conf.em2.bootp_relay = 0
net.ipv4.conf.em2.log_martians = 0
net.ipv4.conf.em2.tag = 0
net.ipv4.conf.em2.arp_filter = 0
net.ipv4.conf.em2.arp_announce = 0
net.ipv4.conf.em2.arp_ignore = 0
net.ipv4.conf.em2.arp_accept = 0
net.ipv4.conf.em2.arp_notify = 0
net.ipv4.conf.em2.proxy_arp_pvlan = 0
net.ipv4.conf.em2.disable_xfrm = 0
net.ipv4.conf.em2.disable_policy = 0
net.ipv4.conf.em2.force_igmp_version = 0
net.ipv4.conf.em2.promote_secondaries = 0
net.ipv4.conf.em2.accept_local = 0
net.ipv4.conf.em3.forwarding = 1
net.ipv4.conf.em3.mc_forwarding = 0
net.ipv4.conf.em3.accept_redirects = 1
net.ipv4.conf.em3.secure_redirects = 1
net.ipv4.conf.em3.shared_media = 1
net.ipv4.conf.em3.rp_filter = 2
net.ipv4.conf.em3.send_redirects = 1
net.ipv4.conf.em3.accept_source_route = 0
net.ipv4.conf.em3.src_valid_mark = 0
net.ipv4.conf.em3.proxy_arp = 0
net.ipv4.conf.em3.medium_id = 0
net.ipv4.conf.em3.bootp_relay = 0
net.ipv4.conf.em3.log_martians = 0
net.ipv4.conf.em3.tag = 0
net.ipv4.conf.em3.arp_filter = 0
net.ipv4.conf.em3.arp_announce = 0
net.ipv4.conf.em3.arp_ignore = 0
net.ipv4.conf.em3.arp_accept = 0
net.ipv4.conf.em3.arp_notify = 0
net.ipv4.conf.em3.proxy_arp_pvlan = 0
net.ipv4.conf.em3.disable_xfrm = 0
net.ipv4.conf.em3.disable_policy = 0
net.ipv4.conf.em3.force_igmp_version = 0
net.ipv4.conf.em3.promote_secondaries = 0
net.ipv4.conf.em3.accept_local = 0
net.ipv4.conf.em1.forwarding = 1
net.ipv4.conf.em1.mc_forwarding = 0
net.ipv4.conf.em1.accept_redirects = 1
net.ipv4.conf.em1.secure_redirects = 1
net.ipv4.conf.em1.shared_media = 1
net.ipv4.conf.em1.rp_filter = 2
net.ipv4.conf.em1.send_redirects = 1
net.ipv4.conf.em1.accept_source_route = 0
net.ipv4.conf.em1.src_valid_mark = 0
net.ipv4.conf.em1.proxy_arp = 0
net.ipv4.conf.em1.medium_id = 0
net.ipv4.conf.em1.bootp_relay = 0
net.ipv4.conf.em1.log_martians = 0
net.ipv4.conf.em1.tag = 0
net.ipv4.conf.em1.arp_filter = 0
net.ipv4.conf.em1.arp_announce = 0
net.ipv4.conf.em1.arp_ignore = 0
net.ipv4.conf.em1.arp_accept = 0
net.ipv4.conf.em1.arp_notify = 0
net.ipv4.conf.em1.proxy_arp_pvlan = 0
net.ipv4.conf.em1.disable_xfrm = 0
net.ipv4.conf.em1.disable_policy = 0
net.ipv4.conf.em1.force_igmp_version = 0
net.ipv4.conf.em1.promote_secondaries = 0
net.ipv4.conf.em1.accept_local = 0
net.ipv4.ip_forward = 1
net.ipv4.xfrm4_gc_thresh = 4194304
net.ipv4.ipfrag_high_thresh = 262144
net.ipv4.ipfrag_low_thresh = 196608
net.ipv4.ipfrag_time = 30
net.ipv4.icmp_echo_ignore_all = 0
net.ipv4.icmp_echo_ignore_broadcasts = 1
net.ipv4.icmp_ignore_bogus_error_responses = 1
net.ipv4.icmp_errors_use_inbound_ifaddr = 0
net.ipv4.icmp_ratelimit = 300
net.ipv4.icmp_ratemask = 6168
net.ipv4.rt_cache_rebuild_count = 4
net.ipv4.ipfrag_secret_interval = 600
net.ipv4.ipfrag_max_dist = 64

agentzh

Hello!

On Fri, Jul 19, 2013 at 3:20 AM, Freeman Zhang wrote:
> 1. 机器配置
>     64GB 内存
>     8核 16线程的 志强处理器
>     运行CentOS 6.3
>
> 2. nginx 承担的工作
>     主要是提供反向代理工作
>     其中有proxy_cache，量大概在600GB
>     HIT命中率在60%左右
>     平常提供QPS 200上下。
>

你能在 TCP 链路上确定到底是谁发起的 RST 包么？

我听说微软的浏览器喜欢主动发起 RST 包来断开请求，我还听说有一些防火墙性质的东西也喜欢发起 RST 包。

你提供的信息过少，所以很难做出有意义的判断。

Regards,
-agentzh