Performance Degraded after upgrade from 2.2.24 container image to 2.2.29

Help!
1

Hello,

Last week we upgraded HAProxy container from 2.2.24 to 2.2.29 (docker pull haproxy:2.2.29) and started seeing slower performance. While probing for details in 2.2.29 logs and comparing with logs from 2.2.24, I found

  1. for non SSL requests the Tc and eventually Tr has increased while there was no change to the backend.
  2. for SSL requests, I see an increased Th as well as Tr.

Is there a way I could enable additional debugs to understand why the Th and Tc are high or any other guidance to approach, understand and fix this performance issue.

FYI, below is my haproxy.cfg

global
maxconn 5000
tune.bufsize 32768
tune.http.logurilen 4096
h1-case-adjust accept-encoding Accept-Encoding
h1-case-adjust content-length Content-Length
h1-case-adjust content-type Content-Type
ssl-default-bind-options ssl-min-ver TLSv1.2
ssl-default-bind-ciphers ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES128-SHA256
log stdout format raw local0

defaults
timeout http-request 15s
timeout queue 30s
timeout connect 5s
timeout client 3m
timeout server 3m
timeout http-keep-alive 10s
retries 6
log global
mode http
option httplog
option dontlognull
option http-keep-alive
option redispatch
option log-health-checks
option http-buffer-request
option forwardfor
option h1-case-adjust-bogus-client
option h1-case-adjust-bogus-server

frontend fe_prod_stats
bind :8500
mode http
stats enable
stats uri /stats
stats refresh 30s

frontend fe_main_https_prod_dc1
bind :8080
bind :8443 ssl crt /usr/local/etc/haproxy/cert/
log global
log-format %ci:%cp\ [%t]\ %ft\ %b/%s\ %Th/%Ti/%TR/%Tq/%Tw/%Tc/%Tr/%Tt\ %ST\ %B\ \ %CC\ %CS\ %tsc\ %ac/%fc/%bc/%sc/%rc\ %sq/%bq\ %hr\ %hs\ %{+Q}r\ %sslc\ %sslv

acl is_app_glb hdr(host) -i abc.xyz.com
acl is_ssl_port dst_port 8443        
acl app_probe dst_port 8080
    
http-request set-header host abc.xyz.com if !is_app_glb
use_backend             bk_http_prod_dc1_probe if app_probe
default_backend bk_https_dc1_prod

backend bk_http_prod_dc1_probe
mode http
balance roundrobin
option httpchk GET /app/probe
http-check expect status 200
default-server inter 2s fall 3 rise 2
server host1_dc1_probe xxx.xxx.xxx.xxx:80
server host2_dc2_probe yyy.yyy.yyy.yyy:80

backend bk_https_dc1_prod
mode http
balance roundrobin
option httpchk GET /app/probe
http-check expect status 200
default-server inter 2s fall 3 rise 2
server host1_dc1_bk xxx.xxx.xxx.xxx:443 check port 80 check ssl verify none
server host2_dc2_bk yyy.yyy.yyy.yyy:443 check port 80 check ssl verify none