Intermittent 503 errors with multiple sites through proxy


#1

I put a new Centos6 haproxy (compiled) system in production with 700+ sites behind it. The load was not high on the proxy or the two backend IIS servers. We were noticing occasional 503 errors on a handful of sites. While getting 503 at the client the site worked directly on the IIS servers. Also I could use curl and wget successfully from haproxy during the issues. After an hour the breif outages were not dissipating and we reverted to our previous load balancer.

Has anyone seen an issue like this that was caused by configuration error or that there is something I can look for?

global
log 127.0.0.1 local2
nbproc 1

chroot /var/lib/haproxy

user haproxy

group haproxy

pidfile                 /var/run/haproxy.pid
maxconn                 90000
spread-checks           2
ssl-server-verify       none
ssl-default-bind-ciphers kEECDH+aRSA+AES:kRSA+AES:+AES256:RC4-SHA:!kEDH:!LOW:!EXP:!MD5:!aNULL:!eNULL
ssl-default-bind-options no-sslv3
daemon
stats socket            /var/lib/haproxy/stats.sock mode 600 level admin
stats timeout 2m

defaults
maxconn 5000
fullconn 5000
mode http
balance roundrobin
log global
option dontlognull

option dontlog-normal

option                  log-separate-errors

option                  http-server-close
option                  abortonclose

option forwardfor except 127.0.0.0/8

option                  redispatch
retries                 3
hash-type consistent

timeout http-request    10s
timeout queue           60s
timeout connect         10s
timeout client          30s
timeout server          30s
timeout http-keep-alive 10s
timeout check           10s

listen stats
bind 192.168.1.254:8080
mode http
stats enable
stats hide-version
stats realm HAProxy\ Statistics
stats uri /hapstats
stats auth haadmin:**********

listen example1.com
bind 192.168.4.22:80 transparent
option httplog
option forwardfor

option httpchk OPTIONS * HTTP/1.1\r\nHost:\ example1.com

stick-table type ip size 5000 expire 20m
stick on src
source                  0.0.0.0 usesrc clientip
server                  192.168.2.17 192.168.2.17:80  minconn 100 maxconn 1000 check inter 5s
server                  192.168.3.17 192.168.3.17:80  minconn 100 maxconn 1000 check inter 5s

listen example2.com
bind 192.168.4.22:80 transparent
option httplog
option forwardfor

option httpchk OPTIONS * HTTP/1.1\r\nHost:\ example2.com

stick-table type ip size 5000 expire 20m
stick on src
source                  0.0.0.0 usesrc clientip
server                  192.168.2.17 192.168.2.17:80  minconn 100 maxconn 1000 check inter 5s
server                  192.168.3.17 192.168.3.17:80  minconn 100 maxconn 1000 check inter 5s

listen fanciersite1.com
bind 192.168.254.71:443 transparent
mode tcp
option tcplog
option tcp-check
stick on src table fanciersite1.com
source 0.0.0.0 usesrc clientip
server 192.168.6.71SSL 192.168.6.71:443 minconn 100 maxconn 1000 check inter 5s
server 192.168.7.71SSL 192.168.7.71:443 minconn 100 maxconn 1000 check inter 5s

listen fanciersite1.com
bind 192.168.254.71:80 transparent
option httplog
option httpchk OPTIONS * HTTP/1.1\r\nHost:\ fanciersite1.com
stick-table type ip size 50k expire 30m
stick on src
source 0.0.0.0 usesrc clientip
server 192.168.6.71 192.168.6.71:80 minconn 100 maxconn 1000 check inter 5s
server 192.168.7.71 192.168.7.71:80 minconn 100 maxconn 1000 check inter 5s

listen fanciersite2.com
bind 192.168.254.73:443 transparent
mode tcp
option tcplog
option tcp-check
stick on src table fanciersite2.com
source 0.0.0.0 usesrc clientip
server 192.168.7.73SSL 192.168.7.73:443 minconn 100 maxconn 1000 check inter 5s
server 192.168.6.73SSL 192.168.6.73:443 minconn 100 maxconn 1000 check inter 5s

listen fanciersite2.com
bind 192.168.254.73:80 transparent
option httplog
option httpchk OPTIONS * HTTP/1.1\r\nHost:\ fanciersite2.com
stick-table type ip size 50k expire 30m
stick on src
source 0.0.0.0 usesrc clientip
server 192.168.7.73 192.168.7.73:80 minconn 100 maxconn 1000 check inter 5s
server 192.168.6.73 192.168.6.73:80 minconn 100 maxconn 1000 check inter 5s


#2

Check haproxy logging. It will tell the exact reason it failed.

Also check the stats page.


#3

Excuse me, has this problem been fix? @webber