Routing problems with multiple domains and tcp/http frontends

[barnybadzoo]

Hi, i use Haproxy 2.0.19 and local letsencrypt plugin at Opnsense Firewall which runs at FreeBsd.

My setup:

• one backend at cloud.mydomain.com for which i need to use ssl termination with local letsencrypt at opnsense.
• one backend which requires for both Ports 80 and 443 TCP called sub1.mydomain.com
• one backend which requires for both Ports 80 and 443 TCP called sub2.mydomain.com, which also should receive all wildcard calls to *.sub2.mydomain.com
• Port 80 should used only for acme redirects and should redirect rest of traffic to ssl
• Currently i redirect calls to port 80 at cloud.domain.com not at haproxy…that i also need to change

My configuration below works well most of the time for the cloud.mydomain.com backend, but fails sometimes with 404 for the tcp backend sub2.mydomain.com.
In the haproxy logfile i couldn’t find an logged message about 404… i could only see in one log that the sub2.domain.com was routed at frontend-in to sub1.domain.com what seems to cause the 404 error. But i could fix that…

I did spent already two days in testing… Please help… How can i get this work…
Thank you…

Here is my config:

global
    uid                         80
    gid                         80
    chroot                      /var/haproxy
    daemon
    stats                       socket /var/run/haproxy.socket group proxy mode 775 level admin
    nbproc                      1
    nbthread                    1
    tune.ssl.default-dh-param   1024
    spread-checks               2
    tune.chksize                16384
    tune.bufsize                16384
    tune.lua.maxmem             0
    log /var/run/log local0 debug
    ssl-default-bind-options no-sslv3 no-tlsv10 no-tls-tickets
    ssl-default-bind-ciphers ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256
    ssl-default-bind-ciphersuites TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256
    

defaults
    log     global
    maxconn 500
    timeout client 5s
    timeout connect 30s
    timeout server 450s
    retries 3
    default-server init-addr libc,last
    mode tcp
    option tcplog

# Frontend: frontend_http ()
frontend frontend_http
    bind *:80 name *:80 
    mode tcp
    default_backend sub1_http
    # tuning options
    timeout client 5s

    # logging options
    option tcplog
    # ACL: find_acme_challenge
    acl acl_5b16dcd0b64453.15125102 path_beg -i /.well-known/acme-challenge/
    # ACL: host_is_cloud
    acl acl_5ae04b3549a228.53262550 hdr(host) -i cloud.mydomain.com

    # ACTION: redirect_acme_challenges
    use_backend acme_challenge_backend if acl_5b16dcd0b64453.15125102
    # ACTION: cloud-domain
    use_backend cloud if acl_5ae04b3549a228.53262550
    # WARNING: pass through options below this line
    option tcplog


# Frontend: frontend_in ()
frontend frontend_in
    bind *:443 name *:443 
    mode tcp
    default_backend sub1_https
    # tuning options
    timeout client 5s
    # stickiness
    stick-table type ip size 50k expire 30m  
    tcp-request connection track-sc0 src
    # logging options
    option tcplog
    # ACL: clienthello
    acl acl_5fff84ac926de8.55606604 req.ssl_hello_type 1 

    # goal here to catch all calls to sub2.mydomain.com and *.sub2.mydomain.com
    # ACL: sub2_sni
    acl acl_60073cbe607c98.79079679 req.ssl_sni -m sub -i sub2.mydomain.com
    # ACL: sub2_sni_2
    acl acl_60074ff27bb5c3.85553936 req.ssl_sni -i sub2.mydomain.com
    # ACL: sub2_sni_3
    acl acl_600758b1975163.29487316 ssl_fc_sni sub2.mydomain.com
    
    # ACL: cloud_sni
    acl acl_60018b049c5c68.40679038 req.ssl_sni -i cloud.mydomain.com
    # ACL: find_acme_challenge
    acl acl_5b16dcd0b64453.15125102 path_beg -i /.well-known/acme-challenge/

    # Inspect Header, capture and log all
    tcp-request inspect-delay 5s 
    tcp-request content capture req.ssl_sni len 100 
    capture request header Referer len 64 
    capture request header Content-Length len 20 
    capture request header User-Agent len 64 
    # ACTION: log_own_format for more details
    # NOTE: actions with no ACLs/conditions will always match
    log-format "frontend-in: ssl-url: %[capture.req.hdr(0)] %{+Q}o\ client_address = %ci, client_port = %cp, server_address = %si, server_port = %sp ,  %Th\ %hr\%hrl \%hsl\ %f\ %b/%s\ %Tw/%Tc/%Tt\ %B\   %tsc\ %ac/%fc/%bc/%sc/%rc\ %sq/%bq\ {%hrl}\ {%hsl}\ " 
    # ACTION: accept_ssl
    tcp-request content accept if acl_5fff84ac926de8.55606604
    # ACTION: sub2_sni
    use_backend sub2 if acl_60073cbe607c98.79079679 || acl_60074ff27bb5c3.85553936 || acl_600758b1975163.29487316
    # ACTION: cloud_sni
    use_backend proxy_ssl if acl_60018b049c5c68.40679038
    # ACTION: redirect_acme_challenges
    use_backend acme_challenge_backend if acl_5b16dcd0b64453.15125102

# Frontend: frontend_https ()
frontend frontend_https
    http-response set-header Strict-Transport-Security "max-age=15552000; includeSubDomains; preload"
    bind 127.0.0.1:4443 name 127.0.0.1:4443 accept-proxy ssl no-sslv3 no-tlsv10 no-tls-tickets ciphers ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256 ciphersuites TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256 crt-list /tmp/haproxy/ssl/60018e42546898.15414025.certlist 
    mode http
    option http-keep-alive
    
    default_backend cloud
    option forwardfor
    # tuning options
    timeout client 5s

    # logging options
    option httplog

    # ACL: find_acme_challenge
    acl acl_5b16dcd0b64453.15125102 path_beg -i /.well-known/acme-challenge/

    # ACTION: redirect_acme_challenges
    use_backend acme_challenge_backend if acl_5b16dcd0b64453.15125102

# Backend: cloud ()
backend cloud
    # health checking is DISABLED
    mode http
    balance source
    # stickiness
    stick-table type ip size 50k expire 30m  
    stick on src
    # tuning options
    timeout connect 30s
    timeout server 450s

    # WARNING: pass through options below this line
    option tcp-smart-connect
    http-reuse never
    server ucs 10.10.10.2:443 ssl verify none

# Backend: acme_challenge_backend (Added by Let's Encrypt plugin)
backend acme_challenge_backend
    # health checking is DISABLED
    mode http
    balance source
    # stickiness
    stick-table type ip size 50k expire 30m  
    stick on src
    # tuning options
    timeout connect 30s
    timeout server 450s
    http-reuse never
    server acme_challenge_host 127.0.0.1:43580 

# Backend: sub1_http ()
backend sub1_http
    # health checking is DISABLED
    mode tcp
    balance source
    # stickiness
    stick-table type ip size 50k expire 30m  
    stick on src
    # tuning options
    timeout connect 30s
    timeout server 450s
    server sub1-http 192.168.1.150:80 

# Backend: sub1_https ()
backend sub1_https
    # health checking is DISABLED
    mode tcp
    balance source
    # stickiness
    stick-table type ip size 50k expire 30m  
    stick on src
    # tuning options
    timeout connect 30s
    timeout server 450s
    server sub1-https 192.168.1.150:443 

# Backend: proxy_ssl ()
backend proxy_ssl
    # health checking is DISABLED
    mode tcp
    balance source
    # stickiness
    stick-table type ip size 50k expire 30m  
    stick on src
    # tuning options
    timeout connect 30s
    timeout server 450s
    server loopback-for-tls 127.0.0.1:4443 send-proxy-v2 check-send-proxy

# Backend: sub2 ()
backend sub2
    # health checking is DISABLED
    mode tcp
    balance source
    # stickiness
    stick-table type ip size 50k expire 30m  
    stick on src
    # tuning options
    timeout connect 30s
    timeout server 450s
    server sub2 192.168.1.180:443 

listen local_statistics
    bind            127.0.0.1:8822
    mode            http
    stats uri       /haproxy?stats
    stats realm     HAProxy\ statistics
    stats admin     if TRUE

[lukastribus]

The certificates must be distinct and must not overlap.

Also see:

Random routing issue (tcp, sni, http/2)

Yes, the symptom is your are connecting to something that belongs on backend X, which works and then the browser needs something from backend Y, but it will reuse the same session that still is going to backend X, because the certificate on backend X covers the hostname that according your rules would actually go to backend Y. They can’t overlap is the point, this is not necessarily related to wildcard certificates, it’s just easier to hit this issue with them. The point is there be a hostname…