Need help to understand VRRP failovers and healthchecks

Help!
1

Hi Team,
New to HAProxy looking for help with the following pair (no network related issues seen i.e spanning-tree re-convergence events etc between a and b LBs)
I’m looking to understand the links from VRRP tracks to haproxy and http what exactly is going wrong for VRRP to be flapping?

haproxy.cfg and keepalived.cfg attached below from both a and b proxies.



sudo cat /etc/keepalived/keepalived.conf

global_defs {
        lvs_id sntdocprdlb01a #Unique name of this Load Balancer
        notification_email {
               technology.solutions@company.co.uk
        }
        notification_email_from sntdocprdlb01a@company.co.uk
        smtp_server 127.0.0.1
        smtp_connect_timeout 30
}

vrrp_script check_haproxy {
        script "/etc/keepalived/check_haproxy.sh"
        interval 1                  #check every 2 seconds
        fall 1
        rise 3
}

vrrp_script check_httpd {
        script "/etc/keepalived/check_httpd.sh"
        interval 1                  #check every 2 seconds
        fall 1
        rise 3
}


vrrp_instance FloatIP01 {
        interface eth0
        track_interface {
                eth0
        }
        state MASTER
        interface eth0
        virtual_router_id 169
        priority 150
        advert_int 1
        smtp_alert
        preempt_delay 300
        unicast_src_ip 10.36.7.167
        unicast_peer {
                10.36.7.168
        }
        authentication {
                auth_type PASS
                auth_pass SN169DOC
        }
        virtual_ipaddress {
                10.36.7.169  dev eth0
        }
        track_script {
                check_haproxy
                check_httpd
        }
        notify_backup "/etc/keepalived/keepalived.state.sh"
        notify_master "/etc/keepalived/keepalived.state.sh"
        notify_fault "/etc/keepalived/keepalived.state.sh"
}















sudo cat /etc/haproxy/haproxy.cfg
#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
    # to have these messages end up in /var/log/haproxy.log you will
    # need to:
    #
    # 1) configure syslog to accept network log events.  This is done
    #    by adding the '-r' option to the SYSLOGD_OPTIONS in
    #    /etc/sysconfig/syslog
    #
    # 2) configure local2 events to go to the /var/log/haproxy.log
    #   file. A line like the following can be added to
    #   /etc/sysconfig/syslog
    #
    #    local2.*                       /var/log/haproxy.log
    #
    log         127.0.0.1 local2
    log         127.0.0.1 local3
    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     10240
    user        haproxy
    group       haproxy
    daemon

    # turn on stats unix socket
    #stats socket /var/lib/haproxy/stats
    stats socket /var/run/info.sock mode 666 level admin
    stats timeout 2m
    stats socket /tmp/haproxy



#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1h
    timeout server          1h
    timeout http-keep-alive 10s
    timeout check           10s
   #maxconn                 3000
    maxconn                 10240
    timeout connect 5000


listen stats :8080
    mode http
    stats enable
    stats hide-version
    stats realm Haproxy\ Statistics
    stats uri /
        stats refresh 60s




#KN hello PRD 25-05-2021
frontend localnode_01-helloPRD
        bind *:9091
        mode tcp
        option tcplog
        default_backend lbServers_01-helloPRD


#NC hello Mongo PRD 09-07-2021
frontend localnode_MongoDB_hello
        bind *:37023
        mode tcp
        option tcplog
        default_backend lbServers_MongoDB_hello

backend lbServers_01-helloPRD
        mode tcp
        stats enable
        #balance source
        balance roundrobin
        #cookie SERVERID insert indirect nocache
        cookie JSESSIONID prefix
        option httpclose
        option forwardfor
        # stick match src
        # stick-table type ip size 200k expire 30m
        #option tcp-check expect string HTTP/1.1 200 OK
        #option tcp-check expect ! string Error

        stick-table type binary len 32 size 30k expire 30m
        acl clienthello req_ssl_hello_type 1
        acl serverhello rep_ssl_hello_type 2

        tcp-request inspect-delay 5s
        tcp-request content accept if clienthello

        tcp-response content accept if clienthello
        stick on payload_lv(43,1) if clienthello
        stick store-response payload_lv(43,1) if serverhello

        server sntdockerprd01b 10.36.7.118:8443 check
        server sntdockerprd02b 10.36.7.119:8443 check
        server sntdockerprd03b 10.36.7.120:8443 check
        server sntdockerprd01a 10.36.7.121:8443 check
        server sntdockerprd02a 10.36.7.122:8443 check






#### MONGO


backend lbServers_MongoDB_hello
        mode tcp
        option tcplog
        stats enable
        option tcp-check
        fullconn 10000
        option tcpka
        balance roundrobin
        server grpsqlneo02a 10.36.7.155:37023 check
        server grpsqlneo03a 10.36.7.156:37023 check
        server grpsqlneo01b 10.36.7.145:37023 check
        server grpsqlneo02b 10.36.7.157:37023 check
        server grpsqlneo03b 10.36.7.158:37023 check





**********************************************************************************************************************************

global_defs {
        lvs_id sntdocprdlb01a #Unique name of this Load Balancer
        notification_email {
               technology.solutions@company.co.uk
        }
        notification_email_from sntdocprdlb01a@company.co.uk
        smtp_server 127.0.0.1
        smtp_connect_timeout 30
}

vrrp_script check_haproxy {
        script "/etc/keepalived/check_haproxy.sh"
        interval 1                  #check every 2 seconds
        fall 1
        rise 3
}

vrrp_script check_httpd {
        script "/etc/keepalived/check_httpd.sh"
        interval 1                  #check every 2 seconds
        fall 1
        rise 3
}


vrrp_instance FloatIP01 {
        interface eth0
        track_interface {
                eth0
        }
        state BACKUP
        interface eth0
        virtual_router_id 169
        priority 100
        advert_int 1
        smtp_alert
        preempt_delay 300
        unicast_src_ip 10.36.7.168
        unicast_peer {
                10.36.7.167
        }
        authentication {
                auth_type PASS
                auth_pass SN169DOC
        }
        virtual_ipaddress {
                10.36.7.169  dev eth0
        }
        track_script {
                check_haproxy
                check_httpd
        }
        notify_backup "/etc/keepalived/keepalived.state.sh"
        notify_master "/etc/keepalived/keepalived.state.sh"
        notify_fault "/etc/keepalived/keepalived.state.sh"
}









sudo cat /etc/haproxy/haproxy.cfg

#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
    # to have these messages end up in /var/log/haproxy.log you will
    # need to:
    #
    # 1) configure syslog to accept network log events.  This is done
    #    by adding the '-r' option to the SYSLOGD_OPTIONS in
    #    /etc/sysconfig/syslog
    #
    # 2) configure local2 events to go to the /var/log/haproxy.log
    #   file. A line like the following can be added to
    #   /etc/sysconfig/syslog
    #
    #    local2.*                       /var/log/haproxy.log
    #
    log         127.0.0.1 local2
    log         127.0.0.1 local3
    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     10240
    user        haproxy
    group       haproxy
    daemon

    # turn on stats unix socket
    #stats socket /var/lib/haproxy/stats
    stats socket /var/run/info.sock mode 666 level admin
    stats timeout 2m
    stats socket /tmp/haproxy



#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1h
    timeout server          1h
    timeout http-keep-alive 10s
    timeout check           10s
   #maxconn                 3000
    maxconn                 10240
    timeout connect 5000


listen stats :8080
    mode http
    stats enable
    stats hide-version
    stats realm Haproxy\ Statistics
    stats uri /
        stats refresh 60s




#KN hello PRD 25-05-2021
frontend localnode_01-helloPRD
        bind *:9091
        mode tcp
        option tcplog
        default_backend lbServers_01-helloPRD


#NC hello Mongo PRD 09-07-2021
frontend localnode_MongoDB_hello
        bind *:37023
        mode tcp
        option tcplog
        default_backend lbServers_MongoDB_hello


####  Back End ###
backend lbServers_01-helloPRD
        mode tcp
        stats enable
        balance roundrobin
        cookie SERVERID insert indirect nocache
        cookie JSESSIONID prefix
        default-server inter 10s fall 2
        # stick match src
        # stick-table type ip size 200k expire 30m

        option tcp-check comment hello_PRD
        option tcplog
        option log-health-checks
        option tcp-check expect string HTTP/1.1 200 OK
        option tcp-check expect ! string Error

        stick-table type binary len 32 size 30k expire 30m
        acl clienthello req_ssl_hello_type 1
        acl serverhello rep_ssl_hello_type 2

        tcp-request inspect-delay 5s
        tcp-request content accept if clienthello

        tcp-response content accept if clienthello
        stick on payload_lv(43,1) if clienthello
        stick store-response payload_lv(43,1) if serverhello

        server sntdockerprd01b 10.36.7.118:8443 check
        server sntdockerprd02b 10.36.7.119:8443 check
        server sntdockerprd03b 10.36.7.120:8443 check
        server sntdockerprd01a 10.36.7.121:8443 check
        server sntdockerprd02a 10.36.7.122:8443 check






#### MONGO


backend lbServers_MongoDB_hello
        mode tcp
        option tcplog
        stats enable
        option tcp-check
        fullconn 10000
        option tcpka
        balance roundrobin
        server grpsqlneo02a 10.36.7.155:37023 check
        server grpsqlneo03a 10.36.7.156:37023 check
        server grpsqlneo01b 10.36.7.145:37023 check
        server grpsqlneo02b 10.36.7.157:37023 check
        server grpsqlneo03b 10.36.7.158:37023 check
2

any help?

3

I’m not sure there are many keepalived experts here.

1 Like