Ssl passthrough(layer4 haproxy) is taking more time in sending the response than ssl termination(layer7 haproxy)

Help!
1

Hi Everyone,

I have a HAProxy server which works at layer7(ssl termination). I have configured the same HAProxy server to layer4(ssl passthrough) to understand the behaviour of HAProxy.

For testing purpose I have written a script which sends 200 concurrent requests to my backend service. Each API request consists a body of size 512KB.

So when haproxy is running in layer7(ssl termination) mode i’m getting response in 2-2.5 minutes… but when haproxy is running is layer4(ssl passthrough) mode i’m getting responses in 6-8 minutes. Could you please tell me the reason for the variation in the response time.

layer7 configuration

frontend fff
  bind *:443 ssl crt server.pem alpn h2,http/1.1
  mode http
  default_backend service_1

backend service_1
  mode http
  option httpchk
  http-check connect ssl alpn h2,http/1.1
  http-check send meth GET uri /health
  server server1 xx.xx.xx.xx:443 check ssl ca-file @system-ca verify required verifyhost <hostname>
  default-server inter 5s

layer4 configuration

frontend fff
  bind *:443 alpn h2,http/1.1
  mode tcp

  # Wait for a client hello for at most 5 seconds
  tcp-request inspect-delay 5s
  tcp-request content accept if { req_ssl_hello_type 1 }
  
  default_backend service_1

backend service_1
  mode tcp
  option httpchk
  http-check connect ssl alpn h2,http/1.1
  http-check send meth GET uri /health
  server server1 xx.xx.xx.xx:443 check check-ssl verify required verifyhost <hostname>
  default-server inter 5s

haproxy -vv

HAProxy version 2.6.12-f588462 2023/03/28 - https://haproxy.org/
Status: long-term supported branch - will stop receiving fixes around Q2 2027.
Known bugs: http://www.haproxy.org/bugs/bugs-2.6.12.html
Running on: Linux 6.2.0-1017-aws #17~22.04.1-Ubuntu SMP Fri Nov 17 21:07:13 UTC 2023 x86_64
Build options :
  TARGET  = linux-glibc
  CPU     = generic
  CC      = cc
  CFLAGS  = -O2 -g -Wall -Wextra -Wundef -Wdeclaration-after-statement -Wfatal-errors -Wtype-limits -Wshift-negative-value -Wshift-overflow=2 -Wduplicated-cond -Wnull-dereference -fwrapv -Wno-address-of-packed-member -Wno-unused-label -Wno-sign-compare -Wno-unused-parameter -Wno-clobbered -Wno-missing-field-initializers -Wno-cast-function-type -Wno-string-plus-int -Wno-atomic-alignment
  OPTIONS = USE_LINUX_TPROXY=1 USE_GETADDRINFO=1 USE_OPENSSL=1 USE_LUA=1 USE_TFO=1 USE_PROMEX=1
  DEBUG   = -DDEBUG_STRICT -DDEBUG_MEMORY_POOLS

Feature list : -51DEGREES +ACCEPT4 +BACKTRACE -CLOSEFROM +CPU_AFFINITY +CRYPT_H -DEVICEATLAS +DL -ENGINE +EPOLL -EVPORTS +GETADDRINFO -KQUEUE +LIBCRYPT +LINUX_SPLICE +LINUX_TPROXY +LUA -MEMORY_PROFILING+NETFILTER +NS -OBSOLETE_LINKER +OPENSSL -OT -PCRE -PCRE2 -PCRE2_JIT -PCRE_JIT +POLL +PRCTL -PROCCTL +PROMEX -QUIC +RT +SLZ -STATIC_PCRE -STATIC_PCRE2 -SYSTEMD +TFO +THREAD +THREAD_DUMP +TPROXY -WURFL -ZLIB

Default settings :
  bufsize = 16384, maxrewrite = 1024, maxpollevents = 200

test script:

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"sync"
	"time"
)


type RandomData struct {
	Value int
}

var counter int
var m sync.Mutex

func updateCounter() int {
	m.Lock()        
	defer m.Unlock() 

	counter = counter + 1

	return counter
}
func main() {
	rand.Seed(time.Now().UnixNano())

	url := "https://<haproxy-serrver-fqdn>/service1/test"
	numEntries := 512 * 1024
	
	// random data generation
	data := make([]RandomData, numEntries)
	for i := range data {
		data[i] = RandomData{Value: rand.Intn(1000)}
	}

	byteData, err := json.Marshal(data)
	if err != nil {
		panic(err)
	}

	// Send concurrent POST requests
	localCounter := 0
	for {
		go sendAPIREquest(url, byteData)

		localCounter += 1
		if localCounter >= 200 {
			break
		}
	}

	http.ListenAndServe(":3334", nil)
}

func sendAPIREquest(url string, jsonData []byte) {

	reqNumber := updateCounter()
	fmt.Printf("request %d sent by client at %s\n", reqNumber, time.Now())
	client := &http.Client{}
	req, err := http.NewRequest("PUT", url, bytes.NewReader(jsonData))
	if err != nil {
		panic(err)
	}
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("accept", "application/json")
	req.Header.Set("reqCount", fmt.Sprintf("%d", reqNumber))

	// Send the request and handle response
	resp, err := client.Do(req)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	fmt.Printf("response of request %d received at %s\n", reqNumber, time.Now())
	_, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		panic(err)
	}
	
	client.CloseIdleConnections()
}

Can anyone please let me know why there is a variation in the response time?

Thank you in advance.