global
log /dev/log local0
log /dev/log local1 notice
user root
group root
daemon
ca-base /etc/ssl/certs
crt-base /etc/ssl/private
ssl-default-bind-ciphers ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA
ssl-default-bind-options no-sslv3
ssl-default-server-ciphers kEECDH+aRSA+AES:kRSA+AES:+AES256:RC4-SHA:!kEDH:!LOW:!EXP:!MD5:!aNULL:!eNULL
ssl-default-server-options no-sslv3
stats socket /run/haproxy.sock mode 660 level admin
defaults
log global
option dontlognull
option redispatch
option tcp-smart-accept
option tcp-smart-connect
timeout connect 5s
timeout client 480m
timeout server 480m
timeout http-keep-alive 1s
timeout http-request 15s
timeout queue 30s
timeout tarpit 1m
frontend mysql
bind <IP>:3306
mode tcp
option tcplog
default_backend mysql_nodes
backend mysql_nodes
mode tcp
balance leastconn
option tcp-check
option httpchk
server mysql-1 <IP1>:3306 backup check port 9200 maxconn 1500 inter 1s fall 5 rise 2
server mysql-2 <IP2>:3306 check port 9200 maxconn 1500 inter 1s fall 5 rise 2
server mysql-3 <IP3>:3306 check port 9200 maxconn 1500 inter 1s fall 5 rise 2
I have set up health check on port 9200 with Xinetd and scripts.
I could see in the log that the layer4 checks are failing and layer7 checks passing fine.
Server mysql_nodes/mysql-2 is DOWN, reason: Layer4 connection problem, info: "Connection refused", check duration: 133ms. 1 active and 1 backup servers left. 2 sessions active, 0 requeued, 0 remaining in queue.
Server mysql_nodes/mysql-2 is DOWN, reason: Layer4 connection problem, info: "Connection refused", check duration: 133ms. 1 active and 1 backup servers left. 2 sessions active, 0 requeued, 0 remaining in queue.
Server mysql_nodes/mysql-2 is UP, reason: Layer7 check passed, code: 200, check duration: 457ms. 2 active and 1 backup servers online. 0 sessions requeued, 0 total in queue.
Server mysql_nodes/mysql-2 is UP, reason: Layer7 check passed, code: 200, check duration: 457ms. 2 active and 1 backup servers online. 0 sessions requeued, 0 total in queue.
This configures only layer7 checks right? Is there a way in the configuration file to disable the layer4 checks?
I tried with tcp-check and mysql-check options but i am getting the same results.
Thanks for any help!!
Ok, can you provide a tcpdump ( tcpdump -i ethX -pns0 -w health-check-traffic.cap host <IP1> and port 9200 ) of the entire health check traffic as well as the output of haproxy -vv.
I checked the tcpdump, although the script is sending “200 ok” there was lot of tcp retransmission and reset in the communication. I am trying to replace the bash script with a python script. I will update after making the changes.
I have recreated the situation with your exact release and configuration and I don’t see any issues.
However looking at your traces again it becomes clear that I was too focused on the HTTP transaction. The haproxy log was correct all along and the traces confirm it:
Sometimes the Port 9200 responds (and then the health check succeeds), and sometimes the TCP handshake to port 9200 is just flat out rejected (SYN → RST, ACK).
