Help with understanding retransmissions from haproxy to the server

Help!
1

Hello, I am fighting with understanding the nature of or retransmissions that are sent after ACK is received.

HA-Proxy version 1.5.18 2016/05/10

Config:

haproxy.cfg

global
log /dev/log local0
log /dev/log local1 debug

daemon
user haproxy
group haproxy
stats socket /var/run/haproxy.sock level admin

defaults
mode tcp
log global
option tcplog
option dontlognull
timeout connect 5s
timeout client 24h
timeout server 60m
maxconn 100000

frontend service_name
bind 50.1.1.3:1234
acl p1234 dst_port 1234
use_backend service_name_48641 if p1234

backend service_name_1234
balance leastconn
option independant-streams
server server_vir1 x1:1234 on-marked-down shutdown-sessions check fall 3 rise 2 inter 10s slowstart 200s source 172.1.2.3
server server_vir2 x2:1234 on-marked-down shutdown-sessions check fall 3 rise 2 inter 10s slowstart 200s source 172.1.2.4
server server_vir3 x3:1234 on-marked-down shutdown-sessions check fall 3 rise 2 inter 10s slowstart 200s source 172.1.2.5

Sysctl config:

sysctl config

abi.vsyscall32 = 1
crypto.fips_enabled = 1
debug.exception-trace = 1
debug.kprobes-optimization = 1
debug.panic_on_rcu_stall = 0
dev.hpet.max-user-freq = 64
fs.aio-max-nr = 65536
fs.aio-nr = 0
fs.nr_open = 1048576
fs.overflowgid = 65534
fs.overflowuid = 65534
fs.pipe-max-size = 1048576
fs.pipe-user-pages-hard = 0
fs.pipe-user-pages-soft = 16384
fs.protected_hardlinks = 1
fs.protected_symlinks = 1
fs.quota.allocated_dquots = 0
fs.quota.cache_hits = 0
fs.quota.drops = 0
fs.quota.free_dquots = 0
fs.quota.lookups = 0
fs.quota.reads = 0
fs.quota.syncs = 4
fs.quota.warnings = 1
fs.quota.writes = 0
fs.suid_dumpable = 2
kernel.random.entropy_avail = 3472
kernel.random.poolsize = 4096
kernel.random.read_wakeup_threshold = 64
kernel.random.urandom_min_reseed_secs = 60
kernel.random.write_wakeup_threshold = 896
kernel.randomize_va_space = 2
kernel.real-root-dev = 0
kernel.sched_autogroup_enabled = 0
kernel.sched_cfs_bandwidth_slice_us = 5000
kernel.sched_child_runs_first = 0
kernel.sched_domain.cpu0.domain0.busy_factor = 32
kernel.sched_domain.cpu0.domain0.busy_idx = 2
kernel.sched_domain.cpu0.domain0.cache_nice_tries = 1
kernel.sched_domain.cpu0.domain0.flags = 4143
kernel.sched_domain.cpu0.domain0.forkexec_idx = 0
kernel.sched_domain.cpu0.domain0.idle_idx = 1
kernel.sched_domain.cpu0.domain0.imbalance_pct = 125
kernel.sched_domain.cpu0.domain0.max_interval = 4
kernel.sched_domain.cpu0.domain0.max_newidle_lb_cost = 17558
kernel.sched_domain.cpu0.domain0.min_interval = 2
kernel.sched_domain.cpu0.domain0.name = DIE
kernel.sched_domain.cpu0.domain0.newidle_idx = 0
kernel.sched_domain.cpu0.domain0.wake_idx = 0
kernel.sched_domain.cpu1.domain0.busy_factor = 32
kernel.sched_domain.cpu1.domain0.busy_idx = 2
kernel.sched_domain.cpu1.domain0.cache_nice_tries = 1
kernel.sched_domain.cpu1.domain0.flags = 4143
kernel.sched_domain.cpu1.domain0.forkexec_idx = 0
kernel.sched_domain.cpu1.domain0.idle_idx = 1
kernel.sched_domain.cpu1.domain0.imbalance_pct = 125
kernel.sched_domain.cpu1.domain0.max_interval = 4
kernel.sched_domain.cpu1.domain0.max_newidle_lb_cost = 9445
kernel.sched_domain.cpu1.domain0.min_interval = 2
kernel.sched_domain.cpu1.domain0.name = DIE
kernel.sched_domain.cpu1.domain0.newidle_idx = 0
kernel.sched_domain.cpu1.domain0.wake_idx = 0
kernel.sched_latency_ns = 12000000
kernel.sched_migration_cost_ns = 500000
kernel.sched_min_granularity_ns = 1500000
kernel.sched_nr_migrate = 32
kernel.sched_rr_timeslice_ms = 100
kernel.sched_rt_period_us = 1000000
kernel.sched_rt_runtime_us = 950000
kernel.sched_schedstats = 0
kernel.sched_shares_window_ns = 10000000
kernel.sched_time_avg_ms = 1000
kernel.sched_tunable_scaling = 1
kernel.sched_wakeup_granularity_ns = 2000000
kernel.sem = 250 32000 32 128
kernel.sem_next_id = -1
kernel.shm_next_id = -1
kernel.shm_rmid_forced = 0
kernel.shmall = 18446744073692774399
kernel.shmmax = 18446744073692774399
kernel.shmmni = 4096
kernel.softlockup_all_cpu_backtrace = 0
kernel.softlockup_panic = 0
kernel.stack_tracer_enabled = 0
kernel.sysrq = 16
kernel.tainted = 0
kernel.threads-max = 62405
kernel.timer_migration = 1
kernel.traceoff_on_warning = 0
kernel.unknown_nmi_panic = 1
kernel.usermodehelper.bset = 4294967295 31
kernel.usermodehelper.inheritable = 4294967295 31
kernel.version = #1 SMP Fri Oct 13 10:46:25 EDT 2017
kernel.watchdog = 1
kernel.watchdog_cpumask = 0-1
kernel.watchdog_thresh = 10
kernel.yama.ptrace_scope = 0
net.core.bpf_jit_enable = 0
net.core.busy_poll = 0
net.core.busy_read = 0
net.core.default_qdisc = pfifo_fast
net.core.dev_weight = 64
net.core.message_burst = 10
net.core.message_cost = 5
net.core.netdev_budget = 300
net.core.netdev_max_backlog = 1000
net.core.netdev_rss_key = 00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00
net.core.netdev_tstamp_prequeue = 1
net.core.optmem_max = 20480
net.core.rmem_default = 212992
net.core.rmem_max = 212992
net.core.rps_sock_flow_entries = 0
net.core.somaxconn = 1024
net.core.warnings = 1
net.core.wmem_default = 212992
net.core.wmem_max = 212992
net.core.xfrm_acq_expires = 30
net.core.xfrm_aevent_etime = 10
net.core.xfrm_aevent_rseqth = 2
net.core.xfrm_larval_drop = 1
net.ipv4.cipso_cache_bucket_size = 10
net.ipv4.cipso_cache_enable = 1
net.ipv4.cipso_rbm_optfmt = 0
net.ipv4.cipso_rbm_strictvalid = 1
net.ipv4.conf.all.accept_local = 0
net.ipv4.conf.all.accept_redirects = 0
net.ipv4.conf.all.accept_source_route = 0
net.ipv4.conf.all.arp_accept = 0
net.ipv4.conf.all.arp_announce = 0
net.ipv4.conf.all.arp_filter = 0
net.ipv4.conf.all.arp_ignore = 0
net.ipv4.conf.all.arp_notify = 0
net.ipv4.conf.all.bootp_relay = 0
net.ipv4.conf.all.disable_policy = 0
net.ipv4.conf.all.disable_xfrm = 0
net.ipv4.conf.all.force_igmp_version = 2
net.ipv4.conf.all.forwarding = 1
net.ipv4.conf.all.log_martians = 1
net.ipv4.conf.all.mc_forwarding = 0
net.ipv4.conf.all.medium_id = 0
net.ipv4.conf.all.promote_secondaries = 1
net.ipv4.conf.all.proxy_arp = 0
net.ipv4.conf.all.proxy_arp_pvlan = 0
net.ipv4.conf.all.route_localnet = 0
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.all.secure_redirects = 0
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.all.shared_media = 1
net.ipv4.conf.all.src_valid_mark = 0
net.ipv4.conf.all.tag = 0
net.ipv4.conf.default.accept_local = 0
net.ipv4.conf.default.accept_redirects = 0
net.ipv4.conf.default.accept_source_route = 0
net.ipv4.conf.default.arp_accept = 0
net.ipv4.conf.default.arp_announce = 0
net.ipv4.conf.default.arp_filter = 0
net.ipv4.conf.default.arp_ignore = 0
net.ipv4.conf.default.arp_notify = 0
net.ipv4.conf.default.bootp_relay = 0
net.ipv4.conf.default.disable_policy = 0
net.ipv4.conf.default.disable_xfrm = 0
net.ipv4.conf.default.force_igmp_version = 2
net.ipv4.conf.default.forwarding = 1
net.ipv4.conf.default.log_martians = 0
net.ipv4.conf.default.mc_forwarding = 0
net.ipv4.conf.default.medium_id = 0
net.ipv4.conf.default.promote_secondaries = 1
net.ipv4.conf.default.proxy_arp = 0
net.ipv4.conf.default.proxy_arp_pvlan = 0
net.ipv4.conf.default.route_localnet = 0
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.default.secure_redirects = 0
net.ipv4.conf.default.send_redirects = 0
net.ipv4.conf.default.shared_media = 1
net.ipv4.conf.default.src_valid_mark = 0
net.ipv4.conf.default.tag = 0
net.ipv4.conf.eth0.accept_local = 0
net.ipv4.conf.eth0.accept_redirects = 0
net.ipv4.conf.eth0.accept_source_route = 0
net.ipv4.conf.eth0.arp_accept = 0
net.ipv4.conf.eth0.arp_announce = 0
net.ipv4.conf.eth0.arp_filter = 0
net.ipv4.conf.eth0.arp_ignore = 0
net.ipv4.conf.eth0.arp_notify = 0
net.ipv4.conf.eth0.bootp_relay = 0
net.ipv4.conf.eth0.disable_policy = 0
net.ipv4.conf.eth0.disable_xfrm = 0
net.ipv4.conf.eth0.force_igmp_version = 0
net.ipv4.conf.eth0.forwarding = 1
net.ipv4.conf.eth0.log_martians = 0
net.ipv4.conf.eth0.mc_forwarding = 0
net.ipv4.conf.eth0.medium_id = 0
net.ipv4.conf.eth0.promote_secondaries = 1
net.ipv4.conf.eth0.proxy_arp = 0
net.ipv4.conf.eth0.proxy_arp_pvlan = 0
net.ipv4.conf.eth0.route_localnet = 0
net.ipv4.conf.eth0.rp_filter = 1
net.ipv4.conf.eth0.secure_redirects = 0
net.ipv4.conf.eth0.send_redirects = 0
net.ipv4.conf.eth0.shared_media = 1
net.ipv4.conf.eth0.src_valid_mark = 0
net.ipv4.conf.eth0.tag = 0
net.ipv4.conf.eth2.accept_local = 0
net.ipv4.conf.eth2.accept_redirects = 0
net.ipv4.conf.eth2.accept_source_route = 0
net.ipv4.conf.eth2.arp_accept = 0
net.ipv4.conf.eth2.arp_announce = 0
net.ipv4.conf.eth2.arp_filter = 0
net.ipv4.conf.eth2.arp_ignore = 0
net.ipv4.conf.eth2.arp_notify = 0
net.ipv4.conf.eth2.bootp_relay = 0
net.ipv4.conf.eth2.disable_policy = 0
net.ipv4.conf.eth2.disable_xfrm = 0
net.ipv4.conf.eth2.force_igmp_version = 0
net.ipv4.conf.eth2.forwarding = 1
net.ipv4.conf.eth2.log_martians = 0
net.ipv4.conf.eth2.mc_forwarding = 0
net.ipv4.conf.eth2.medium_id = 0
net.ipv4.conf.eth2.promote_secondaries = 1
net.ipv4.conf.eth2.proxy_arp = 0
net.ipv4.conf.eth2.proxy_arp_pvlan = 0
net.ipv4.conf.eth2.route_localnet = 0
net.ipv4.conf.eth2.rp_filter = 1
net.ipv4.conf.eth2.secure_redirects = 0
net.ipv4.conf.eth2.send_redirects = 0
net.ipv4.conf.eth2.shared_media = 1
net.ipv4.conf.eth2.src_valid_mark = 0
net.ipv4.conf.eth2.tag = 0
net.ipv4.conf.lo.accept_local = 0
net.ipv4.conf.lo.accept_redirects = 1
net.ipv4.conf.lo.accept_source_route = 1
net.ipv4.conf.lo.arp_accept = 0
net.ipv4.conf.lo.arp_announce = 0
net.ipv4.conf.lo.arp_filter = 0
net.ipv4.conf.lo.arp_ignore = 0
net.ipv4.conf.lo.arp_notify = 0
net.ipv4.conf.lo.bootp_relay = 0
net.ipv4.conf.lo.disable_policy = 1
net.ipv4.conf.lo.disable_xfrm = 1
net.ipv4.conf.lo.force_igmp_version = 0
net.ipv4.conf.lo.forwarding = 1
net.ipv4.conf.lo.log_martians = 0
net.ipv4.conf.lo.mc_forwarding = 0
net.ipv4.conf.lo.medium_id = 0
net.ipv4.conf.lo.promote_secondaries = 0
net.ipv4.conf.lo.proxy_arp = 0
net.ipv4.conf.lo.proxy_arp_pvlan = 0
net.ipv4.conf.lo.route_localnet = 0
net.ipv4.conf.lo.rp_filter = 0
net.ipv4.conf.lo.secure_redirects = 1
net.ipv4.conf.lo.send_redirects = 1
net.ipv4.conf.lo.shared_media = 1
net.ipv4.conf.lo.src_valid_mark = 0
net.ipv4.conf.lo.tag = 0
net.ipv4.icmp_echo_ignore_all = 0
net.ipv4.icmp_echo_ignore_broadcasts = 1
net.ipv4.icmp_errors_use_inbound_ifaddr = 0
net.ipv4.icmp_ignore_bogus_error_responses = 1
net.ipv4.icmp_msgs_burst = 50
net.ipv4.icmp_msgs_per_sec = 1000
net.ipv4.icmp_ratelimit = 1000
net.ipv4.icmp_ratemask = 6168
net.ipv4.igmp_max_memberships = 20
net.ipv4.igmp_max_msf = 10
net.ipv4.igmp_qrv = 2
net.ipv4.inet_peer_maxttl = 600
net.ipv4.inet_peer_minttl = 120
net.ipv4.inet_peer_threshold = 65664
net.ipv4.ip_default_ttl = 64
net.ipv4.ip_dynaddr = 0
net.ipv4.ip_early_demux = 1
net.ipv4.ip_forward = 1
net.ipv4.ip_forward_use_pmtu = 0
net.ipv4.ip_local_port_range = 1024 65535
net.ipv4.ip_local_reserved_ports =
net.ipv4.ip_no_pmtu_disc = 0
net.ipv4.ip_nonlocal_bind = 1
net.ipv4.ipfrag_high_thresh = 4194304
net.ipv4.ipfrag_low_thresh = 3145728
net.ipv4.ipfrag_max_dist = 64
net.ipv4.ipfrag_secret_interval = 600
net.ipv4.ipfrag_time = 30
net.ipv4.neigh.default.anycast_delay = 100
net.ipv4.neigh.default.app_solicit = 0
net.ipv4.neigh.default.base_reachable_time_ms = 30000
net.ipv4.neigh.default.delay_first_probe_time = 5
net.ipv4.neigh.default.gc_interval = 30
net.ipv4.neigh.default.gc_stale_time = 60
net.ipv4.neigh.default.gc_thresh1 = 128
net.ipv4.neigh.default.gc_thresh2 = 512
net.ipv4.neigh.default.gc_thresh3 = 1024
net.ipv4.neigh.default.locktime = 100
net.ipv4.neigh.default.mcast_solicit = 3
net.ipv4.neigh.default.proxy_delay = 80
net.ipv4.neigh.default.proxy_qlen = 64
net.ipv4.neigh.default.retrans_time_ms = 1000
net.ipv4.neigh.default.ucast_solicit = 3
net.ipv4.neigh.default.unres_qlen = 31
net.ipv4.neigh.default.unres_qlen_bytes = 65536
net.ipv4.neigh.eth0.anycast_delay = 100
net.ipv4.neigh.eth0.app_solicit = 0
net.ipv4.neigh.eth0.base_reachable_time_ms = 30000
net.ipv4.neigh.eth0.delay_first_probe_time = 5
net.ipv4.neigh.eth0.gc_stale_time = 60
net.ipv4.neigh.eth0.locktime = 100
net.ipv4.neigh.eth0.mcast_solicit = 3
net.ipv4.neigh.eth0.proxy_delay = 80
net.ipv4.neigh.eth0.proxy_qlen = 64
net.ipv4.neigh.eth0.retrans_time_ms = 1000
net.ipv4.neigh.eth0.ucast_solicit = 3
net.ipv4.neigh.eth0.unres_qlen = 31
net.ipv4.neigh.eth0.unres_qlen_bytes = 65536
net.ipv4.neigh.eth1.anycast_delay = 100
net.ipv4.neigh.eth1.app_solicit = 0
net.ipv4.neigh.eth1.base_reachable_time_ms = 30000
net.ipv4.neigh.eth1.delay_first_probe_time = 5
net.ipv4.neigh.eth1.gc_stale_time = 60
net.ipv4.neigh.eth1.locktime = 100
net.ipv4.neigh.eth1.mcast_solicit = 3
net.ipv4.neigh.eth1.proxy_delay = 80
net.ipv4.neigh.eth1.proxy_qlen = 64
net.ipv4.neigh.eth1.retrans_time_ms = 1000
net.ipv4.neigh.eth1.ucast_solicit = 3
net.ipv4.neigh.eth1.unres_qlen = 31
net.ipv4.neigh.eth1.unres_qlen_bytes = 65536
net.ipv4.neigh.eth2.anycast_delay = 100
net.ipv4.neigh.eth2.app_solicit = 0
net.ipv4.neigh.eth2.base_reachable_time_ms = 30000
net.ipv4.neigh.eth2.delay_first_probe_time = 5
net.ipv4.neigh.eth2.gc_stale_time = 60
net.ipv4.neigh.eth2.locktime = 100
net.ipv4.neigh.eth2.mcast_solicit = 3
net.ipv4.neigh.eth2.proxy_delay = 80
net.ipv4.neigh.eth2.proxy_qlen = 64
net.ipv4.neigh.eth2.retrans_time_ms = 1000
net.ipv4.neigh.eth2.ucast_solicit = 3
net.ipv4.neigh.eth2.unres_qlen = 31
net.ipv4.neigh.eth2.unres_qlen_bytes = 65536
net.ipv4.neigh.lo.anycast_delay = 100
net.ipv4.neigh.lo.app_solicit = 0
net.ipv4.neigh.lo.base_reachable_time_ms = 30000
net.ipv4.neigh.lo.delay_first_probe_time = 5
net.ipv4.neigh.lo.gc_stale_time = 60
net.ipv4.neigh.lo.locktime = 100
net.ipv4.neigh.lo.mcast_solicit = 3
net.ipv4.neigh.lo.proxy_delay = 80
net.ipv4.neigh.lo.proxy_qlen = 64
net.ipv4.neigh.lo.retrans_time_ms = 1000
net.ipv4.neigh.lo.ucast_solicit = 3
net.ipv4.neigh.lo.unres_qlen = 31
net.ipv4.neigh.lo.unres_qlen_bytes = 65536
net.ipv4.ping_group_range = 1 0
net.ipv4.route.error_burst = 5000
net.ipv4.route.error_cost = 1000
net.ipv4.route.gc_elasticity = 8
net.ipv4.route.gc_interval = 60
net.ipv4.route.gc_min_interval = 0
net.ipv4.route.gc_min_interval_ms = 500
net.ipv4.route.gc_thresh = -1
net.ipv4.route.gc_timeout = 300
net.ipv4.route.max_size = 2147483647
net.ipv4.route.min_adv_mss = 256
net.ipv4.route.min_pmtu = 552
net.ipv4.route.mtu_expires = 600
net.ipv4.route.redirect_load = 20
net.ipv4.route.redirect_number = 9
net.ipv4.route.redirect_silence = 20480
net.ipv4.tcp_abort_on_overflow = 0
net.ipv4.tcp_adv_win_scale = 1
net.ipv4.tcp_allowed_congestion_control = cubic reno
net.ipv4.tcp_app_win = 31
net.ipv4.tcp_autocorking = 1
net.ipv4.tcp_available_congestion_control = cubic reno
net.ipv4.tcp_base_mss = 1024
net.ipv4.tcp_challenge_ack_limit = 2147483647
net.ipv4.tcp_congestion_control = cubic
net.ipv4.tcp_dsack = 1
net.ipv4.tcp_early_retrans = 3
net.ipv4.tcp_ecn = 2
net.ipv4.tcp_fack = 1
net.ipv4.tcp_fastopen = 0
net.ipv4.tcp_fastopen_key = 00000000-00000000-00000000-00000000
net.ipv4.tcp_fin_timeout = 60
net.ipv4.tcp_frto = 2
net.ipv4.tcp_invalid_ratelimit = 500
net.ipv4.tcp_keepalive_intvl = 75
net.ipv4.tcp_keepalive_probes = 9
net.ipv4.tcp_keepalive_time = 7200
net.ipv4.tcp_limit_output_bytes = 262144
net.ipv4.tcp_low_latency = 0
net.ipv4.tcp_max_orphans = 32768
net.ipv4.tcp_max_ssthresh = 0
net.ipv4.tcp_max_syn_backlog = 1280
net.ipv4.tcp_max_tw_buckets = 32768
net.ipv4.tcp_mem = 185361 247148 370722
net.ipv4.tcp_min_tso_segs = 2
net.ipv4.tcp_moderate_rcvbuf = 1
net.ipv4.tcp_mtu_probing = 2
net.ipv4.tcp_no_metrics_save = 0
net.ipv4.tcp_notsent_lowat = -1
net.ipv4.tcp_orphan_retries = 0
net.ipv4.tcp_reordering = 3
net.ipv4.tcp_retrans_collapse = 1
net.ipv4.tcp_retries1 = 3
net.ipv4.tcp_retries2 = 15
net.ipv4.tcp_rfc1337 = 0
net.ipv4.tcp_rmem = 4096 87380 6291456
net.ipv4.tcp_sack = 1
net.ipv4.tcp_slow_start_after_idle = 0
net.ipv4.tcp_stdurg = 0
net.ipv4.tcp_syn_retries = 6
net.ipv4.tcp_synack_retries = 5
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_thin_dupack = 0
net.ipv4.tcp_thin_linear_timeouts = 0
net.ipv4.tcp_timestamps = 1
net.ipv4.tcp_tso_win_divisor = 3
net.ipv4.tcp_tw_recycle = 0
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_wmem = 4096 16384 4194304
net.ipv4.tcp_workaround_signed_windows = 0
net.ipv4.udp_mem = 187218 249624 374436
net.ipv4.udp_rmem_min = 4096
net.ipv4.udp_wmem_min = 4096
net.ipv4.vs.am_droprate = 10
net.ipv4.vs.amemthresh = 1024
net.ipv4.vs.backup_only = 0
net.ipv4.vs.cache_bypass = 0
net.ipv4.vs.conn_reuse_mode = 1
net.ipv4.vs.conntrack = 0
net.ipv4.vs.drop_entry = 0
net.ipv4.vs.drop_packet = 0
net.ipv4.vs.expire_nodest_conn = 0
net.ipv4.vs.expire_quiescent_template = 0
net.ipv4.vs.nat_icmp_send = 0
net.ipv4.vs.pmtu_disc = 1
net.ipv4.vs.secure_tcp = 0
net.ipv4.vs.snat_reroute = 1
net.ipv4.vs.sync_ports = 1
net.ipv4.vs.sync_qlen_max = 61800
net.ipv4.vs.sync_refresh_period = 0
net.ipv4.vs.sync_retries = 0
net.ipv4.vs.sync_sock_size = 0
net.ipv4.vs.sync_threshold = 3 50
net.ipv4.vs.sync_version = 1
net.ipv4.xfrm4_gc_thresh = 32768
net.netfilter.nf_conntrack_acct = 0
net.netfilter.nf_conntrack_buckets = 65536
net.netfilter.nf_conntrack_checksum = 1
net.netfilter.nf_conntrack_count = 7575
net.netfilter.nf_conntrack_dccp_loose = 1
net.netfilter.nf_conntrack_dccp_timeout_closereq = 64
net.netfilter.nf_conntrack_dccp_timeout_closing = 64
net.netfilter.nf_conntrack_dccp_timeout_open = 43200
net.netfilter.nf_conntrack_dccp_timeout_partopen = 480
net.netfilter.nf_conntrack_dccp_timeout_request = 240
net.netfilter.nf_conntrack_dccp_timeout_respond = 480
net.netfilter.nf_conntrack_dccp_timeout_timewait = 240
net.netfilter.nf_conntrack_events = 1
net.netfilter.nf_conntrack_events_retry_timeout = 15
net.netfilter.nf_conntrack_expect_max = 1024
net.netfilter.nf_conntrack_generic_timeout = 600
net.netfilter.nf_conntrack_helper = 1
net.netfilter.nf_conntrack_icmp_timeout = 30
net.netfilter.nf_conntrack_log_invalid = 0
net.netfilter.nf_conntrack_max = 1048576
net.netfilter.nf_conntrack_sctp_timeout_closed = 10
net.netfilter.nf_conntrack_sctp_timeout_cookie_echoed = 3
net.netfilter.nf_conntrack_sctp_timeout_cookie_wait = 3
net.netfilter.nf_conntrack_sctp_timeout_established = 432000
net.netfilter.nf_conntrack_sctp_timeout_heartbeat_acked = 210
net.netfilter.nf_conntrack_sctp_timeout_heartbeat_sent = 30
net.netfilter.nf_conntrack_sctp_timeout_shutdown_ack_sent = 3
net.netfilter.nf_conntrack_sctp_timeout_shutdown_recd = 0
net.netfilter.nf_conntrack_sctp_timeout_shutdown_sent = 0
net.netfilter.nf_conntrack_tcp_be_liberal = 0
net.netfilter.nf_conntrack_tcp_loose = 0
net.netfilter.nf_conntrack_tcp_max_retrans = 3
net.netfilter.nf_conntrack_tcp_timeout_close = 10
net.netfilter.nf_conntrack_tcp_timeout_close_wait = 60
net.netfilter.nf_conntrack_tcp_timeout_established = 3600
net.netfilter.nf_conntrack_tcp_timeout_fin_wait = 120
net.netfilter.nf_conntrack_tcp_timeout_last_ack = 30
net.netfilter.nf_conntrack_tcp_timeout_max_retrans = 300
net.netfilter.nf_conntrack_tcp_timeout_syn_recv = 60
net.netfilter.nf_conntrack_tcp_timeout_syn_sent = 120
net.netfilter.nf_conntrack_tcp_timeout_time_wait = 120
net.netfilter.nf_conntrack_tcp_timeout_unacknowledged = 300
net.netfilter.nf_conntrack_timestamp = 0
net.netfilter.nf_conntrack_udp_timeout = 30
net.netfilter.nf_conntrack_udp_timeout_stream = 180
net.netfilter.nf_log.0 = NONE
net.netfilter.nf_log.1 = NONE
net.netfilter.nf_log.2 = nfnetlink_log
net.netfilter.nf_log.3 = NONE
net.netfilter.nf_log.4 = NONE
net.netfilter.nf_log.5 = NONE
net.netfilter.nf_log.6 = NONE
net.netfilter.nf_log.7 = NONE
net.netfilter.nf_log.8 = NONE
net.netfilter.nf_log.9 = NONE
net.nf_conntrack_max = 1048576
net.unix.max_dgram_qlen = 512
sunrpc.max_resvport = 1023
sunrpc.min_resvport = 665
sunrpc.nfs_debug = 0x0000
sunrpc.nfsd_debug = 0x0000
sunrpc.nlm_debug = 0x0000
sunrpc.rpc_debug = 0x0000
sunrpc.tcp_fin_timeout = 15
sunrpc.tcp_max_slot_table_entries = 65536
sunrpc.tcp_slot_table_entries = 2
sunrpc.transports = tcp 1048576
sunrpc.transports = udp 32768
sunrpc.transports = tcp-bc 1048576
sunrpc.udp_slot_table_entries = 16
user.max_ipc_namespaces = 31202
user.max_mnt_namespaces = 31202
user.max_net_namespaces = 31202
user.max_pid_namespaces = 31202
user.max_user_namespaces = 0
user.max_uts_namespaces = 31202
vm.admin_reserve_kbytes = 8192
vm.block_dump = 0
vm.dirty_background_bytes = 0
vm.dirty_background_ratio = 10
vm.dirty_bytes = 0
vm.dirty_expire_centisecs = 3000
vm.dirty_ratio = 20
vm.dirty_writeback_centisecs = 500
vm.drop_caches = 0
vm.extfrag_threshold = 500
vm.hugepages_treat_as_movable = 0
vm.hugetlb_shm_group = 0
vm.laptop_mode = 0
vm.legacy_va_layout = 0
vm.lowmem_reserve_ratio = 256 256 32
vm.max_map_count = 65530
vm.memory_failure_early_kill = 0
vm.memory_failure_recovery = 1
vm.min_free_kbytes = 67584
vm.min_slab_ratio = 5
vm.min_unmapped_ratio = 1
vm.mmap_min_addr = 4096
vm.mmap_rnd_bits = 28
vm.mmap_rnd_compat_bits = 8
vm.nr_hugepages = 0
vm.nr_hugepages_mempolicy = 0
vm.nr_overcommit_hugepages = 0
vm.nr_pdflush_threads = 0
vm.numa_zonelist_order = default
vm.oom_dump_tasks = 1
vm.oom_kill_allocating_task = 0
vm.overcommit_kbytes = 0
vm.overcommit_memory = 0
vm.overcommit_ratio = 50
vm.page-cluster = 3
vm.panic_on_oom = 0
vm.percpu_pagelist_fraction = 0
vm.stat_interval = 1
vm.swappiness = 60
vm.user_reserve_kbytes = 131072
vm.vfs_cache_pressure = 100
vm.zone_reclaim_mode = 0

hostnamectl

Icon name: computer-vm
Chassis: vm
Virtualization: vmware
Operating System: Red Hat Enterprise Linux Server 7.5 (Maipo)
CPE OS Name: cpe:/o:redhat:enterprise_linux:7.5:GA:server
Kernel: Linux 3.10.0-862.11.6.el7.x86_64
Architecture: x86-64

Retransmissions

“No.”,“Time”,“Source”,“Destination”,“Delta”,“Protocol”,“Length”,“Seq”,“Next seq”,“ack”,“MSS Value”,“Info”
“1”,“0.000000”,“1.2.3.4”,“50.1.1.3”,“0.000000”,“TCP”,“76”,“0”,“0”,“0”,“1420”,“40935 > 1234 [SYN] Seq=0 Win=65535 Len=0 MSS=1420 SACK_PERM=1 WS=256”
“2”,“0.000021”,“50.1.1.3”,“1.2.3.4”,“0.000021”,“TCP”,“76”,“0”,“0”,“1”,“1460”,“1234 > 40935 [SYN, ACK] Seq=0 Ack=1 Win=28960 Len=0 MSS=1460 SACK_PERM=1 WS=128”
“3”,“0.040704”,“1.2.3.4”,“50.1.1.3”,“0.040683”,“TCP”,“68”,“1”,“1”,“1”,“”,“40935 > 1234 [ACK] Seq=1 Ack=1 Win=85248 Len=0”
“4”,“0.040825”,“172.1.2.6”,“172.1.2.10”,“0.000121”,“TCP”,“76”,“0”,“0”,“0”,“1460”,“16712 > 1234 [SYN] Seq=0 Win=29200 Len=0 MSS=1460 SACK_PERM=1 WS=128”
“5”,“0.040881”,“172.1.2.10”,“172.1.2.6”,“0.000056”,“TCP”,“76”,“0”,“0”,“1”,“1460”,“1234 > 16712 [SYN, ACK] Seq=0 Ack=1 Win=28960 Len=0 MSS=1460 SACK_PERM=1 WS=128”
“6”,“0.040894”,“172.1.2.6”,“172.1.2.10”,“0.000013”,“TCP”,“68”,“1”,“1”,“1”,“”,“16712 > 1234 [ACK] Seq=1 Ack=1 Win=29312 Len=0”
“7”,“0.043899”,“1.2.3.4”,“50.1.1.3”,“0.003005”,“TLSv1.2”,“276”,“1”,“209”,“1”,“”,“Client Hello”
“8”,“0.043914”,“50.1.1.3”,“1.2.3.4”,“0.000015”,“TCP”,“68”,“1”,“1”,“209”,“”,“1234 > 40935 [ACK] Seq=1 Ack=209 Win=30080 Len=0”
“9”,“0.043967”,“172.1.2.6”,“172.1.2.10”,“0.000053”,“TLSv1.2”,“276”,“1”,“209”,“1”,“”,“Client Hello”
“10”,“0.044020”,“172.1.2.10”,“172.1.2.6”,“0.000053”,“TCP”,“68”,“1”,“1”,“209”,“”,“1234 > 16712 [ACK] Seq=1 Ack=209 Win=30080 Len=0”
“11”,“0.044497”,“172.1.2.10”,“172.1.2.6”,“0.000477”,“TLSv1.2”,“154”,“1”,“87”,“209”,“”,“Server Hello”
“12”,“0.044506”,“172.1.2.6”,“172.1.2.10”,“0.000009”,“TCP”,“68”,“209”,“209”,“87”,“”,“16712 > 1234 [ACK] Seq=209 Ack=87 Win=29312 Len=0”
“13”,“0.044532”,“50.1.1.3”,“1.2.3.4”,“0.000026”,“TLSv1.2”,“154”,“1”,“87”,“209”,“”,“Server Hello”
“14”,“0.044541”,“172.1.2.10”,“172.1.2.6”,“0.000009”,“TLSv1.2”,“119”,“87”,“138”,“209”,“”,“Change Cipher Spec, Encrypted Handshake Message”
“15”,“0.044545”,“172.1.2.6”,“172.1.2.10”,“0.000004”,“TCP”,“68”,“209”,“209”,“138”,“”,“16712 > 1234 [ACK] Seq=209 Ack=138 Win=29312 Len=0”
“16”,“0.044561”,“50.1.1.3”,“1.2.3.4”,“0.000016”,“TLSv1.2”,“119”,“87”,“138”,“209”,“”,“Change Cipher Spec, Encrypted Handshake Message”
“17”,“0.072641”,“1.2.3.4”,“50.1.1.3”,“0.028080”,“TCP”,“68”,“209”,“209”,“87”,“”,“40935 > 1234 [ACK] Seq=209 Ack=87 Win=85248 Len=0”
“18”,“0.080670”,“1.2.3.4”,“50.1.1.3”,“0.008029”,“TCP”,“68”,“209”,“209”,“138”,“”,“40935 > 1234 [ACK] Seq=209 Ack=138 Win=85248 Len=0”
“19”,“0.080893”,“1.2.3.4”,“50.1.1.3”,“0.000223”,“TLSv1.2”,“119”,“209”,“260”,“138”,“”,“Change Cipher Spec, Encrypted Handshake Message”
“20”,“0.080941”,“172.1.2.6”,“172.1.2.10”,“0.000048”,“TLSv1.2”,“119”,“209”,“260”,“138”,“”,“Change Cipher Spec, Encrypted Handshake Message”
“21”,“0.091404”,“1.2.3.4”,“50.1.1.3”,“0.010463”,“TCP”,“1476”,“260”,“1668”,“138”,“”,“40935 > 1234 [ACK] Seq=260 Ack=138 Win=85248 Len=1408 [TCP segment of a reassembled PDU]”
“22”,“0.091417”,“50.1.1.3”,“1.2.3.4”,“0.000013”,“TCP”,“68”,“138”,“138”,“1668”,“”,“1234 > 40935 [ACK] Seq=138 Ack=1668 Win=33024 Len=0”
“23”,“0.091475”,“172.1.2.6”,“172.1.2.10”,“0.000058”,“TCP”,“1476”,“260”,“1668”,“138”,“”,“16712 > 1234 [PSH, ACK] Seq=260 Ack=138 Win=29312 Len=1408 [TCP segment of a reassembled PDU]”
“24”,“0.091565”,“172.1.2.10”,“172.1.2.6”,“0.000090”,“TCP”,“68”,“138”,“138”,“1668”,“”,“1234 > 16712 [ACK] Seq=138 Ack=1668 Win=33024 Len=0”
“25”,“0.131937”,“1.2.3.4”,“50.1.1.3”,“0.040372”,“TLSv1.2”,“332”,“1668”,“1932”,“138”,“”,“Application Data”
“26”,“0.131964”,“172.1.2.6”,“172.1.2.10”,“0.000027”,“TLSv1.2”,“332”,“1668”,“1932”,“138”,“”,“Application Data”
“27”,“0.136742”,“172.1.2.10”,“172.1.2.6”,“0.004778”,“TLSv1.2”,“297”,“138”,“367”,“1932”,“”,“Application Data”
“28”,“0.136765”,“50.1.1.3”,“1.2.3.4”,“0.000023”,“TLSv1.2”,“297”,“138”,“367”,“1932”,“”,“Application Data”
“29”,“0.176645”,“172.1.2.6”,“172.1.2.10”,“0.039880”,“TCP”,“68”,“1932”,“1932”,“367”,“”,“16712 > 1234 [ACK] Seq=1932 Ack=367 Win=30336 Len=0”
“30”,“0.212685”,“1.2.3.4”,“50.1.1.3”,“0.036040”,“TCP”,“68”,“1932”,“1932”,“367”,“”,“40935 > 1234 [ACK] Seq=1932 Ack=367 Win=86272 Len=0”
“31”,“200.137180”,“172.1.2.10”,“172.1.2.6”,“199.924495”,“TLSv1.2”,“99”,“367”,“398”,“1932”,“”,“Encrypted Alert”
“32”,“200.137196”,“172.1.2.6”,“172.1.2.10”,“0.000016”,“TCP”,“68”,“1932”,“1932”,“398”,“”,“16712 > 1234 [ACK] Seq=1932 Ack=398 Win=30336 Len=0”
“33”,“200.137214”,“172.1.2.10”,“172.1.2.6”,“0.000018”,“TCP”,“68”,“398”,“398”,“1932”,“”,“1234 > 16712 [FIN, ACK] Seq=398 Ack=1932 Win=35840 Len=0”
“34”,“200.137233”,“50.1.1.3”,“1.2.3.4”,“0.000019”,“TLSv1.2”,“99”,“367”,“398”,“1932”,“”,“Encrypted Alert”
“35”,“200.137292”,“50.1.1.3”,“1.2.3.4”,“0.000059”,“TCP”,“68”,“398”,“398”,“1932”,“”,“1234 > 40935 [FIN, ACK] Seq=398 Ack=1932 Win=35840 Len=0”
“36”,“200.168647”,“1.2.3.4”,“50.1.1.3”,“0.031355”,“TCP”,“68”,“1932”,“1932”,“398”,“”,“40935 > 1234 [ACK] Seq=1932 Ack=398 Win=86272 Len=0”
“37”,“200.176660”,“172.1.2.6”,“172.1.2.10”,“0.008013”,“TCP”,“68”,“1932”,“1932”,“399”,“”,“16712 > 1234 [ACK] Seq=1932 Ack=399 Win=30336 Len=0”
“38”,“200.198702”,“1.2.3.4”,“50.1.1.3”,“0.022042”,“TCP”,“68”,“1932”,“1932”,“399”,“”,“40935 > 1234 [ACK] Seq=1932 Ack=399 Win=86272 Len=0”
“39”,“300.204869”,“1.2.3.4”,“50.1.1.3”,“100.006167”,“TLSv1.2”,“99”,“1932”,“1963”,“399”,“”,“Encrypted Alert”
“40”,“300.204896”,“172.1.2.6”,“172.1.2.10”,“0.000027”,“TLSv1.2”,“99”,“1932”,“1963”,“399”,“”,“Encrypted Alert”
“41”,“300.207748”,“1.2.3.4”,“50.1.1.3”,“0.002852”,“TCP”,“68”,“1963”,“1963”,“399”,“”,“40935 > 1234 [RST, ACK] Seq=1963 Ack=399 Win=86272 Len=0”
“42”,“300.207785”,“172.1.2.6”,“172.1.2.10”,“0.000037”,“TCP”,“68”,“1963”,“1963”,“399”,“”,“16712 > 1234 [FIN, ACK] Seq=1963 Ack=399 Win=30336 Len=0”
“43”,“300.207837”,“172.1.2.10”,“172.1.2.6”,“0.000052”,“TCP”,“68”,“399”,“399”,“1964”,“”,“1234 > 16712 [ACK] Seq=399 Ack=1964 Win=35712 Len=0”
“44”,“300.217641”,“172.1.2.6”,“172.1.2.10”,“0.009804”,“TCP”,“68”,“1963”,“1963”,“399”,“”,“[TCP Retransmission] 16712 > 1234 [FIN, ACK] Seq=1963 Ack=399 Win=30336 Len=0”
“45”,“300.217690”,“172.1.2.10”,“172.1.2.6”,“0.000049”,“TCP”,“68”,“399”,“399”,“1964”,“”,“[TCP Dup ACK 43#1] 1234 > 16712 [ACK] Seq=399 Ack=1964 Win=35712 Len=0”
“46”,“300.419643”,“172.1.2.6”,“172.1.2.10”,“0.201953”,“TLSv1.2”,“99”,“1932”,“1964”,“399”,“”,“[TCP Spurious Retransmission] , Encrypted Alert”
“47”,“300.419704”,“172.1.2.10”,“172.1.2.6”,“0.000061”,“TCP”,“68”,“399”,“399”,“1964”,“”,“[TCP Dup ACK 43#2] 1234 > 16712 [ACK] Seq=399 Ack=1964 Win=35712 Len=0”
“48”,“300.824654”,“172.1.2.6”,“172.1.2.10”,“0.404950”,“TLSv1.2”,“99”,“1932”,“1964”,“399”,“”,“[TCP Spurious Retransmission] , Encrypted Alert”
“49”,“300.824709”,“172.1.2.10”,“172.1.2.6”,“0.000055”,“TCP”,“68”,“399”,“399”,“1964”,“”,“[TCP Dup ACK 43#3] 1234 > 16712 [ACK] Seq=399 Ack=1964 Win=35712 Len=0”
“50”,“301.633648”,“172.1.2.6”,“172.1.2.10”,“0.808939”,“TLSv1.2”,“99”,“1932”,“1964”,“399”,“”,“[TCP Spurious Retransmission] , Encrypted Alert”
“51”,“301.633705”,“172.1.2.10”,“172.1.2.6”,“0.000057”,“TCP”,“68”,“399”,“399”,“1964”,“”,“[TCP Dup ACK 43#4] 1234 > 16712 [ACK] Seq=399 Ack=1964 Win=35712 Len=0”
“52”,“303.253642”,“172.1.2.6”,“172.1.2.10”,“1.619937”,“TLSv1.2”,“99”,“1932”,“1964”,“399”,“”,“[TCP Spurious Retransmission] , Encrypted Alert”
“53”,“303.253698”,“172.1.2.10”,“172.1.2.6”,“0.000056”,“TCP”,“68”,“399”,“399”,“1964”,“”,“[TCP Dup ACK 43#5] 1234 > 16712 [ACK] Seq=399 Ack=1964 Win=35712 Len=0”
“54”,“306.493641”,“172.1.2.6”,“172.1.2.10”,“3.239943”,“TLSv1.2”,“99”,“1932”,“1964”,“399”,“”,“[TCP Spurious Retransmission] , Encrypted Alert”
“55”,“306.493694”,“172.1.2.10”,“172.1.2.6”,“0.000053”,“TCP”,“68”,“399”,“399”,“1964”,“”,“[TCP Dup ACK 43#6] 1234 > 16712 [ACK] Seq=399 Ack=1964 Win=35712 Len=0”
“56”,“312.965655”,“172.1.2.6”,“172.1.2.10”,“6.471961”,“TLSv1.2”,“99”,“1932”,“1964”,“399”,“”,“[TCP Spurious Retransmission] , Encrypted Alert”
“57”,“312.965775”,“172.1.2.10”,“172.1.2.6”,“0.000120”,“TCP”,“68”,“399”,“399”,“1964”,“”,“[TCP Dup ACK 43#7] 1234 > 16712 [ACK] Seq=399 Ack=1964 Win=35712 Len=0”
“58”,“325.925671”,“172.1.2.6”,“172.1.2.10”,“12.959896”,“TLSv1.2”,“99”,“1932”,“1964”,“399”,“”,“[TCP Spurious Retransmission] , Encrypted Alert”
“59”,“325.925726”,“172.1.2.10”,“172.1.2.6”,“0.000055”,“TCP”,“68”,“399”,“399”,“1964”,“”,“[TCP Dup ACK 43#8] 1234 > 16712 [ACK] Seq=399 Ack=1964 Win=35712 Len=0”
“60”,“351.845660”,“172.1.2.6”,“172.1.2.10”,“25.919934”,“TLSv1.2”,“99”,“1932”,“1964”,“399”,“”,“[TCP Spurious Retransmission] , Encrypted Alert”
“61”,“351.845697”,“172.1.2.10”,“172.1.2.6”,“0.000037”,“TCP”,“68”,“399”,“399”,“1964”,“”,“[TCP Dup ACK 43#9] 1234 > 16712 [ACK] Seq=399 Ack=1964 Win=35712 Len=0”

after haproxy got RST form the client (frame 41)
connection between client and haproxy seems to be closd
after haproxy sends FIN,ACK to the server ( frame 42 )
and gets ACK from the server ( frame 43 )

But starting from “frame 44” haproxy starts to send retransmissions of FIN,ACK like it never got ACK from the server. It seems like ACK had been arrived to the interface but never arrived to the process. or I am missing some important config line or I it is a bug?
Help me to understand the nature of those retransmissions.

2

The kernel handles everything TCP related, not haproxy.

Can you share an actual pcap file?

3

Correct, my assumption is that ACK I see in the network capture received by the kernel but never reach haproxy or there some logic in haproxy that ignores that ACK…
Right now I can’t really share pcap file… Need to work on it to hide ips

4

What I am saying is that ACKs or FIN’s are never delivered to an application like haproxy. That’s the kernel jobs to handle them. The interface between kernel and haproxy is the socket API, it’s not a TCP stack.

The application can influence TCP behavior a little, but for the most part the kernel handles it on it’s own.

5

Understand, thanks for clarification.

What are the possible reasons for kernel to send retransmissions after it got ACK? I checked pcap and all retrnasmissions have different identifier so it is not a loop on a vswitch

6

The proxy instance continues to send 31 bytes of TLS encrypted alert towards that backend, which is already correctly acknowledged. Wireshark shows this as Spurious Retransmission which is correct.

I think your local firewall may be dropping the TCP ACK signal because of the close_wait timeout.

Can you bump net.netfilter.nf_conntrack_tcp_timeout_close_wait from 60 to something way beyond 100, actually make it 300 just for testing.

7

Thanks, seems like it resolved retranmission problem. setting it to 110 was enough

1 Like
8

With this particular client, that closed his side of the connection after 100 seconds. Another client may keep the TLS sessions open longer.

I’d suggest you don’t firewall your internal network between haproxy and the backend servers.

9

I can’t disable iptables completely as it is used to filter incoming traffic on public interface and also used to set DSCP bits on some packets.
Is there any way to disable netfilter, connection tracking on private network only? I need to study this.

10

I guess you’d disable connection-tracking and accept all traffic on that specific interface in both the input and output chain, something like:

iptables -I INPUT 1 -i safeeth -j NOTRACK
iptables -I INPUT 2 -i safeeth -j ACCEPT
iptables -I OUTPUT 1 -i safeeth -j NOTRACK
iptables -I OUTPUT 2 -i safeeth -j ACCEPT

But make sure you properly understand what this does and test it (including rollback, box reboot, etc).

11

Thanks, I think I can test this, I will also test it with default net.netfilter.nf_conntrack_tcp_timeout_close_wait 60
Seems like disabling tracking on private network will be safer than tuning conntrack parameters.