HAProxy community

Haproxy 1.9.0 segfault at 7f141e6e3ab8 ip 00007f141e6e3ab8 sp 00007ffea3eab4b8 error 15 in libc-2.17.so[7f141e6e3000+2000]


#11

Does it crash immediately with the first SSL connection or does it work for some time and crash then?


#12

Can you guys please apply the following 4 patches (to either 1.9 or master) in the following tarball and retry:

https://dts.ltri.eu/d.php/e742fb25967bb68db8a6cfcaaa796c4d/19-safaricrash.tar


#13

still got coredump after apply the 4 patches (patch 0002 empty).

(gdb) bt full
#0  0x0000000000000000 in ?? ()
No symbol table info available.
#1  0x000000000051bf1a in connect_server (s=0x32b7480) at src/backend.c:1258
        sess = 0x309ac20
        cli_conn = 0x0
        srv_conn = 0x376ec20
        old_conn = 0x38a8788
        srv_cs = 0x0
        srv = 0x293ca40
        reuse = 1
        reuse_orphan = 0
        err = 0
#2  0x0000000000461a57 in sess_update_stream_int (s=0x32b7480) at src/stream.c:928
        conn_err = 0
        srv = 0x293ca40
        si = 0x32b7758
        req = 0x32b7490
#3  0x0000000000465ad7 in process_stream (t=0x32b7870, context=0x32b7480, state=1025) at src/stream.c:2305
        srv = 0x293ca40
        s = 0x32b7480
        sess = 0x2d6cd60
        rqf_last = 209715202
        rpf_last = 2147483648
        rq_prod_last = 7
        rq_cons_last = 0
        rp_cons_last = 7
        rp_prod_last = 0
        req_ana_back = 32768
        req = 0x32b7490
        res = 0x32b74f0
        si_f = 0x32b7718
        si_b = 0x32b7758
#4  0x0000000000573a4c in process_runnable_tasks () at src/task.c:432
        t = 0x32b7870
        state = 1025
        ctx = 0x32b7480
        process = 0x46332d <process_stream>
        t = 0x2e457e0
        max_processed = 200
#5  0x00000000004ba385 in run_poll_loop () at src/haproxy.c:2619
        next = -226266230
        exp = -226266296
#6  0x00000000004ba704 in run_thread_poll_loop (data=0x2931f40) at src/haproxy.c:2684
        ptif = 0xb49360 <per_thread_init_list>
        ptdf = 0x0
        start_lock = 0
#7  0x00000000004bbf3e in main (argc=6, argv=0x7ffd52f2a6f8) at src/haproxy.c:3313
        tids = 0x2931f40
        threads = 0x2b504a0
        i = 1
        old_sig = {__val = {0, 139834890709042, 29, 139834894120800, 24, 42968336, 30208, 11586480, 2, 0, 0, 0, 0, 0, 0, 0}}
        blocked_sig = {__val = {18446744067199990583, 18446744073709551615 <repeats 15 times>}}
        err = 0
        retry = 200
        limit = {rlim_cur = 400091, rlim_max = 400091}
        errmsg = "\000\244\362R\375\177\000\000\000\000\000\000\000\000\000\000|", '\000' <repeats 15 times>, "|\000\000\000\000\000\000\000`\367\060\331-\177\000\000\030\000\000\000\000\000\000\000\200\066\216\002\000\000\000\000>\001\000\024\000\000\000\000\260\367\260\000\000\000\000\000`a\217\002\000\000\000\000\254\341\374\330-\177\000\000\260\245\362R"
        pidfd = 6

#14

Something went wrong with the tarball, patch 2 must not be empty.

I reuploaded the tarball and double checked that all patches are there:

https://dts.ltri.eu/d.php/f0c88d2e04bf1f2ef8dfbbe84c3027ee/safaripatches2.tar


#15

now got the 4 patches but still coredump as below.

(gdb) bt full
#0  0x0000000000000000 in ?? ()
No symbol table info available.
#1  0x000000000051bf1a in connect_server (s=0x2c61600) at src/backend.c:1258
        sess = 0x2c001f0
        cli_conn = 0x0
        srv_conn = 0x2bc5f80
        old_conn = 0x24ee408
        srv_cs = 0x0
        srv = 0x1c59460
        reuse = 1
        reuse_orphan = 0
        err = 0
#2  0x0000000000461a57 in sess_update_stream_int (s=0x2c61600) at src/stream.c:928
        conn_err = 0
        srv = 0x1c59460
        si = 0x2c618d8
        req = 0x2c61610
#3  0x0000000000465ad7 in process_stream (t=0x2c619f0, context=0x2c61600, state=1025) at src/stream.c:2305
        srv = 0x1c59460
        s = 0x2c61600
        sess = 0x223fc80
        rqf_last = 209715202
        rpf_last = 2147483648
        rq_prod_last = 7
        rq_cons_last = 0
        rp_cons_last = 7
        rp_prod_last = 0
        req_ana_back = 32768
        req = 0x2c61610
        res = 0x2c61670
        si_f = 0x2c61898
        si_b = 0x2c618d8
#4  0x0000000000573a4c in process_runnable_tasks () at src/task.c:432
        t = 0x2c619f0
        state = 1025
        ctx = 0x2c61600
        process = 0x46332d <process_stream>
        t = 0x2f62cb0
        max_processed = 195
#5  0x00000000004ba385 in run_poll_loop () at src/haproxy.c:2619
        next = -200494007
        exp = -200494064
---Type <return> to continue, or q <return> to quit---
#6  0x00000000004ba704 in run_thread_poll_loop (data=0x1bf5f40) at src/haproxy.c:2684
        ptif = 0xb49360 <per_thread_init_list>
        ptdf = 0x0
        start_lock = 0
#7  0x00000000004bbf3e in main (argc=6, argv=0x7ffd72f92b48) at src/haproxy.c:3313
        tids = 0x1bf5f40
        threads = 0x1be9c20
        i = 1
        old_sig = {__val = {0, 0, 29, 139802521089888, 24, 29091088, 30208, 11586480, 140726532385664, 140726532385608, 6, 6329987, 140726532384968, 13, 2, 0}}
        blocked_sig = {__val = {18446744067199990583, 18446744073709551615 <repeats 15 times>}}
        err = 0
        retry = 200
        limit = {rlim_cur = 400091, rlim_max = 400091}
        errmsg = "\000)\371r\375\177\000\000\000\000\000\000\000\000\000\000|", '\000' <repeats 15 times>, "|\000\000\000\000\000\000\000`\267\233O&\177\000\000\030\000\000\000\000\000\000\000\200v\272\001\000\000\000\000>\001\000\024\000\000\000\000\260\367\260\000\000\000\000\000`\241\273\001\000\000\000\000\254\241gO&\177\000\000\000*\371r"
        pidfd = 6

#16

Nope. It is crashing few moments later.


Haproxy 1.8.4 segfault at 8 ip 0000000000475cb2
#17

Please apply the following patch on top of the 4 patches above (but not the patch I send to @safari privately), so the the 4 patches in the tarball and then the following patch:

diff --git a/src/backend.c b/src/backend.c
index 39b40587..4be61585 100644
--- a/src/backend.c
+++ b/src/backend.c
@@ -1158,7 +1158,7 @@ int connect_server(struct stream *s)
 				srv_list = LIST_ELEM(s->sess->srv_list.n,
 						struct sess_srv_list *, srv_list);
 				if (!LIST_ISEMPTY(&srv_list->srv_list))
-					srv_conn = LIST_ELEM(srv_list->srv_list.n,
+					srv_conn = LIST_ELEM(srv_list->conn_list.n,
 						struct connection *, session_list);
 			}
 		}
-- 
2.14.4

This should fix the issue.


#18

@lukastribus, unfortunately, I still got the coredump.


#19

Ok, just to confirm, you are doing a clean rebuild with make clean before recompiling, right? Just trying to make 100% that we are using solid conclusions, sorry if it’s a stupid question.


#20

What I did are: remove the haproxy folder, unzip the haproxy tarball, apply the 4-patches, then the latest patch, make and run.


#21

Do you have any hints about what we should try to reproduce your issue ? How many servers should we need, SSL or not to the server, htx or not, H1 or H2 to servers, any particular setting of http-reuse, etc. Every such thing would be really helpful. Also, if you have any hint about the rough number of requests the process supports before crashing, it would help us figure what type of test to focus on (i.e. if it crashes from the second request, no need to run 1 million request through each config).

Thanks!


#22

Please can you try again with the patches from this patch (to be applied to a naked master) : http://people.freebsd.org/~cognet/haproxy-patches.tar.gz

Thanks!


#23

Hi Willy, the latest patch works fine now. Seems that the “0005-MEDIUM-servers-Be-smarter-when-switching-connections.patch” solve our coredump problem.

Everything’s back to normal with us.
Tks so much.


Balance uri / consistent hashing / redispatch 3, not redispatching?
#24

thank you, the patches were merged into master now. They’ll be backported into the next 1.9 (very soon hopefully).


#25

I believe I’m suffering from similar errors using HAProxy for ssl termination—I see similar segfaults with similar error codes in syslog. I’m running Ubunut 16.04LTS and using the haproxy package by Vincent Bernat (via the debian team site), so I don’t know how useful I’ll be if you guys need any additional information.

It sounds like if my issue is the same as @safari, I should just sit tight and wait for the fix to show up in 1.9 stable and for the package maintainer to update the package from there.

If you guys need/want me to submit any logs or troubleshooting info, please let me know—happy to help in any way possible.


#26

We’ve fixed a significant number of bugs and I’ll emit 1.9.1 on tuesday 8th, please try again with it (or right now with the 1.9 maintenance branch if you want).


#27

1.8.17 and 1.9.1 have been released with the fix for this bug.


#28

@lukastribus still crashing

[14516844.371937] haproxy[32302]: segfault at 8 ip 000000000042aca5 sp 00007ffdfd1dbef0 error 4 in haproxy[400000+183000]
[14516844.474076] haproxy[32303]: segfault at 8 ip 000000000042aca5 sp 00007ffdfd1dbef0 error 4 in haproxy[400000+183000]
[14516844.610569] haproxy[32304]: segfault at 8 ip 000000000042aca5 sp 00007ffdfd1dbef0 error 4 in haproxy[400000+183000]
[14516844.719870] haproxy[32301]: segfault at 8 ip 000000000042aca5 sp 00007ffdfd1dbef0 error 4 in haproxy[400000+183000]
[14516915.466312] haproxy[32478]: segfault at 8 ip 000000000042aca5 sp 00007ffd34174aa0 error 4 in haproxy[400000+183000]
[14516941.380339] haproxy[32588]: segfault at 8 ip 00000000004f70ee sp 00007fffda6e6720 error 4 in haproxy[400000+183000]
[14517156.564684] haproxy[1149]: segfault at 8 ip 000000000042ac55 sp 00007ffdb7ab6a20 error 4 in haproxy[400000+183000]

./haproxy -vvv

HA-Proxy version 1.9.1 2019/01/08 - https://haproxy.org/
Build options :
TARGET = linux2628
CPU = generic
CC = gcc
CFLAGS = -m64 -march=x86-64 -O2 -g -fno-strict-aliasing -Wdeclaration-after-statement -fwrapv -Wno-unused-label -Wno-sign-compare -Wno-unused-parameter -Wno-old-style-declaration -Wno-ignored-qualifiers -Wno-clobbered -Wno-missing-field-initializers -Wtype-limits
OPTIONS = USE_ZLIB=1 USE_CPU_AFFINITY=1 USE_OPENSSL=1 USE_PCRE=1 USE_TFO=1

Default settings :
maxconn = 2000, bufsize = 16384, maxrewrite = 1024, maxpollevents = 200

Built with OpenSSL version : OpenSSL 1.0.1k-fips 8 Jan 2015
Running on OpenSSL version : OpenSSL 1.0.1k-fips 8 Jan 2015
OpenSSL library supports TLS extensions : yes
OpenSSL library supports SNI : yes
OpenSSL library supports : SSLv3 TLSv1.0 TLSv1.1 TLSv1.2
Built with transparent proxy support using: IP_TRANSPARENT IPV6_TRANSPARENT IP_FREEBIND
Built with zlib version : 1.2.8
Running on zlib version : 1.2.8
Compression algorithms supported : identity(“identity”), deflate(“deflate”), raw-deflate(“deflate”), gzip(“gzip”)
Built with PCRE version : 8.21 2011-12-12
Running on PCRE version : 8.21 2011-12-12
PCRE library supports JIT : no (USE_PCRE_JIT not set)
Encrypted password support via crypt(3): yes
Built with multi-threading support.

Available polling systems :
epoll : pref=300, test result OK
poll : pref=200, test result OK
select : pref=150, test result OK
Total: 3 (3 usable), will use epoll.

Available multiplexer protocols :
(protocols marked as cannot be specified using ‘proto’ keyword)
h2 : mode=HTX side=FE|BE
h2 : mode=HTTP side=FE
: mode=HTX side=FE|BE
: mode=TCP|HTTP side=FE|BE

Available filters :
[SPOE] spoe
[COMP] compression
[CACHE] cache
[TRACE] trace


#29

Provide a backtrace as explained in post #2 please.


#30

GNU gdb (GDB) Amazon Linux (7.6.1-64.33.amzn1)
Copyright © 2013 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later http://gnu.org/licenses/gpl.html
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type “show copying”
and “show warranty” for details.
This GDB was configured as “x86_64-amazon-linux-gnu”.
For bug reporting instructions, please see:
http://www.gnu.org/software/gdb/bugs/
Reading symbols from /tmp/haproxy…done.
[New LWP 32487]
[Thread debugging using libthread_db enabled]
Using host libthread_db library “/lib64/libthread_db.so.1”.
Core was generated by `./haproxy -d -db -f /etc/haproxy/haproxy.cfg’.
Program terminated with signal 11, Segmentation fault.
#0 0x000000000055b90e in srv_add_to_idle_list (srv=0x1d12950, conn=0x3c7c660) at include/proto/server.h:244
244 LIST_ADDQ(&srv->idle_orphan_conns[tid], &conn->list);
(gdb) bt
#0 0x000000000055b90e in srv_add_to_idle_list (srv=0x1d12950, conn=0x3c7c660) at include/proto/server.h:244
#1 0x000000000055cdaf in session_free (sess=0x2d8a8e0) at src/session.c:90
#2 0x000000000055cea7 in conn_session_free (conn=0x2d8a730) at src/session.c:112
#3 0x0000000000579cfc in mux_pt_destroy (ctx=0x2d8a9b0) at src/mux_pt.c:38
#4 0x000000000057a0ff in mux_pt_detach (cs=0x2d8aa50) at src/mux_pt.c:193
#5 0x000000000045dab0 in cs_destroy (cs=0x2d8aa50) at include/proto/connection.h:708
#6 0x000000000045e2f3 in si_release_endpoint (si=0x2d8ad38) at include/proto/stream_interface.h:170
#7 0x000000000045fac2 in stream_free (s=0x2d8aaa0) at src/stream.c:446
#8 0x000000000046619d in process_stream (t=0x2d8ae90, context=0x2d8aaa0, state=1025) at src/stream.c:2610
#9 0x0000000000574d37 in process_runnable_tasks () at src/task.c:432
#10 0x00000000004b9cb8 in run_poll_loop () at src/haproxy.c:2619
#11 0x00000000004ba037 in run_thread_poll_loop (data=0x23f91b0) at src/haproxy.c:2684
#12 0x00000000004bb871 in main (argc=5, argv=0x7ffd28d76d58) at src/haproxy.c:3313
(gdb) bt full
#0 0x000000000055b90e in srv_add_to_idle_list (srv=0x1d12950, conn=0x3c7c660) at include/proto/server.h:244
No locals.
#1 0x000000000055cdaf in session_free (sess=0x2d8a8e0) at src/session.c:90
conn = 0x3c7c660
conn_back = 0x3c7c7b8
srv_list = 0x3c7c810
srv_list_back = 0x2d8a968
#2 0x000000000055cea7 in conn_session_free (conn=0x2d8a730) at src/session.c:112
No locals.
#3 0x0000000000579cfc in mux_pt_destroy (ctx=0x2d8a9b0) at src/mux_pt.c:38
conn = 0x2d8a730
#4 0x000000000057a0ff in mux_pt_detach (cs=0x2d8aa50) at src/mux_pt.c:193
conn = 0x2d8a730
ctx = 0x2d8a9b0
#5 0x000000000045dab0 in cs_destroy (cs=0x2d8aa50) at include/proto/connection.h:708
No locals.
#6 0x000000000045e2f3 in si_release_endpoint (si=0x2d8ad38) at include/proto/stream_interface.h:170
conn = 0x0
cs = 0x2d8aa50
appctx = 0x0
#7 0x000000000045fac2 in stream_free (s=0x2d8aaa0) at src/stream.c:446
sess = 0x2d8a8e0
fe = 0x1cc29b0
—Type to continue, or q to quit—
bref = 0x2d8abc8
back = 0x2d8abc8
cli_cs = 0x2d8aa50
must_free_sess = 0
i = 0
#8 0x000000000046619d in process_stream (t=0x2d8ae90, context=0x2d8aaa0, state=1025) at src/stream.c:2610
srv = 0x0
s = 0x2d8aaa0
sess = 0x2d8a8e0
rqf_last = 12640288
rpf_last = 2151719008
rq_prod_last = 9
rq_cons_last = 9
rp_cons_last = 9
rp_prod_last = 9
req_ana_back = 0
req = 0x2d8aab0
res = 0x2d8ab10
si_f = 0x2d8ad38
si_b = 0x2d8ad78
#9 0x0000000000574d37 in process_runnable_tasks () at src/task.c:432
t = 0x2d8ae90
state = 1025
—Type to continue, or q to quit—
ctx = 0x2d8aaa0
process = 0x462977 <process_stream>
t = 0x3cac700
max_processed = 73
#10 0x00000000004b9cb8 in run_poll_loop () at src/haproxy.c:2619
next = 951060494
exp = 951060049
#11 0x00000000004ba037 in run_thread_poll_loop (data=0x23f91b0) at src/haproxy.c:2684
ptif = 0x82fa40 <per_thread_init_list>
ptdf = 0x0
start_lock = 0
#12 0x00000000004bb871 in main (argc=5, argv=0x7ffd28d76d58) at src/haproxy.c:3313
tids = 0x23f91b0
threads = 0x23f7170
i = 1
old_sig = {__val = {0, 140575788085425, 140724603453564, 140725288660312, 18446603344811131004, 2, 0, 0, 18446603348420891937, 2, 18446603348420891921, 2, 0, 0,
390842023984, 140725288659696}}
blocked_sig = {__val = {18446744067199990583, 18446744073709551615 <repeats 15 times>}}
err = 0
retry = 200
limit = {rlim_cur = 104445, rlim_max = 104445}
errmsg = “\000m\327(\375\177\000\000Xm\327(\375\177\000\000\005\000\000\000\000\000\000\000l\262\356Y\332\177\000\000[\001\000\000\000\000\000\000\030\000\000\000\000\000\000\000\315e\177Z\332\177\000\000\230\002oZ\332\177\000\000 \206\315Z\332\177\000\000@\221\312\001\230\001\000\000\362i\177Z\332\177\000\000\001\000\000\000\n\000\000\00—Type to continue, or q to quit—
0\210m\327(”
pidfd = -1