Intermittent net/http: TLS handshake timeout

Help!
1

Hi , All of a sudden working cluster seeing TLS handshake timeout’s not sure where I messed up. I am seriously hung here… Can you please help

Below is the failing curl
[root@xx-01 ~]# curl -k -s -D- https://lb_ip:8443/console -vvv

  • About to connect() to ip port 8443 (#0)
  • Trying ip…
  • Connected to lb_ip (lb_ip) port 8443 (#0)
  • Initializing NSS with certpath: sql:/etc/pki/nssdb

Below is the working curl( its purely intermittent , sometimes it works and sometimes it wont)

curl -k -s -D- https://lb_ip -vvv

  • About to connect() to lb_ip port 443 (#0) * Trying lb_ip… * Connected to lb_ip (lb_ip) port 443 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * skipping SSL peer certificate verification * SSL connection using TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 * Server certificate: * subject: MAIL=mail_id,UID=000,CN=hostname,OU=CEDP_Infra,O=ibm.com,L=Zurich,ST=Zurich,C=CH * start date: Aug 21 04:00:00 2019 GMT * expire date: Nov 23 04:59:59 2021 GMT * common name: hostname * issuer: CN=xxINTERNAL INTERMEDIATE CA,O=XA,C=US > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: lb_ip > Accept: /

Below is the haproxy.cfg file

global
# To view messages in the /var/log/haproxy.log you need to:
#
# 1) Configure syslog to accept network log events. This is done
# by adding the ‘-r’ option to the SYSLOGD_OPTIONS in
# /etc/sysconfig/syslog.
#
# 2) Configure local2 events to go to the /var/log/haproxy.log
# file. A line similar to the following can be added to
# /etc/sysconfig/syslog.
#
# local2.* /var/log/haproxy.log
#
log 127.0.0.1 local0
tune.ssl.default-dh-param 2048
daemon
maxconn 5000
maxsslconn 5000
tune.maxrewrite 32768
tune.bufsize 65536
tune.ssl.default-dh-param 2048

  chroot      /var/lib/haproxy
  pidfile     /var/run/haproxy.pid
  user        haproxy
  group       haproxy
  user        foobar
  stats socket /var/run/haproxy.stat

  # 3) Turn on stats unix socket
  stats socket /var/lib/haproxy/stats

Common defaults that all the ‘listen’ and ‘backend’ sections

use, if not designated in their block.

defaults
mode http
log global
option httplog
option dontlognull
option http-server-close
option redispatch
retries 3
timeout http-request 10s
timeout queue 1m
timeout connect 10s
timeout client 5m
timeout server 5m
timeout http-keep-alive 20s
timeout check 10s
balance source
hash-type consistent
timeout client-fin 60s
timeout server-fin 60s

frontend stats
bind *:8404
stats enable
stats uri /stats
stats refresh 10s
stats admin if LOCALHOST

frontend k8s-api *:8001
mode tcp
option tcplog
use_backend k8s-api

backend k8s-api
mode tcp
option httpchk GET /
http-check expect status 401 # Not Authorizedd
server master-01 xx:8001 check inter 5s fall 3 rise 2 check-ssl verify none
server master-02 xx:8001 check inter 5s fall 3 rise 2 check-ssl verify none
server master-03 xx:8001 check inter 5s fall 3 rise 2 check-ssl verify none

frontend dashboard *:8443
mode tcp
option tcplog
use_backend dashboard

backend dashboard
mode tcp
option httpchk GET /
http-check expect status 302 # Redirect
server master-01 xx:8443 check inter 5s fall 3 rise 2 check-ssl verify none
server master-02 xx:8443 check inter 5s fall 3 rise 2 check-ssl verify none
server master-03 xx:8443 check inter 5s fall 3 rise 2 check-ssl verify none

frontend auth *:9443
mode tcp
option tcplog
use_backend auth

backend auth
mode tcp
option tcp-check
server master-01 xx:9443 check inter 5s fall 3 rise 2 check-ssl verify none
server master-02 xx:9443 check inter 5s fall 3 rise 2 check-ssl verify none
server master-03 xx:9443 check inter 5s fall 3 rise 2 check-ssl verify none

frontend registry *:8500
mode tcp
option tcplog
use_backend registry

backend registry
mode tcp
option httpchk GET /
server master-01 xx:8500 check inter 5s fall 3 rise 2 check-ssl verify none
server master-02 xx:8500 check inter 5s fall 3 rise 2 check-ssl verify none
server master-03 xx:8500 check inter 5s fall 3 rise 2 check-ssl verify none

frontend image-manager *:8600
mode tcp
option tcplog
use_backend image-manager

backend image-manager
mode tcp
option httpchk GET /
server master-01 xx:8600 check inter 5s fall 3 rise 2 check-ssl verify none
server master-02 xx:8600 check inter 5s fall 3 rise 2 check-ssl verify none
server master-03 xx:8600 check inter 5s fall 3 rise 2 check-ssl verify none

frontend proxy-nodeport *:30000-32767
mode tcp
option tcplog
use_backend proxy-nodeport

backend proxy-nodeport
mode tcp
option tcp-check
tcp-check connect
server proxy-01 xx check port 443 inter 5s fall 3 rise 2 check-ssl verify none
server proxy-02 xx check port 443 inter 5s fall 3 rise 2 check-ssl verify none
server proxy-03 xx check port 443 inter 5s fall 3 rise 2 check-ssl verify none

frontend proxy-https *:443
mode tcp
option tcplog
use_backend proxy-https

backend proxy-https
mode tcp
option httpchk GET /badpath/
http-check expect status 404 # Not Found
server proxy-01 xx:443 check inter 5s fall 3 rise 2 check-ssl verify none
server proxy-02 xx:443 check inter 5s fall 3 rise 2 check-ssl verify none
server proxy-03 xx:443 check inter 5s fall 3 rise 2 check-ssl verify none