IronBee in HAProxy


#1

Hi everyone,

The joined patch is an integration of ironBee in HAProxy. This is absolutely experimental.
I’m not able to configure IronBee, so I just run some simple checks.

On my first test, I saw some blocking access initiated by ironbee. These access are to
/dev/random. Maybe these access can be disabled, but I don’t known the way to do this.

The configuration is easy, you can just include this line:

http-request deny if { ironbee(ironbee.waf.conf) -m bool }

“ironbee” is a sample fetch which returns true if an attach is detected.

Its easy to compile, after patching the last HAProxy version, you can compile with these options:

make ... \
   USE_IRONBEE=1 \
   IRONBEE_INC="/opt/ironbee/include" \
   IRONBEE_INC="/opt/ironbee/lib"

Here the patch:
Download link

From 6b2910c3321a6632f5d7f67a35825502c9ae78a0 Mon Sep 17 00:00:00 2001
From: Thierry FOURNIER <tfournier@arpalert.org>
Date: Wed, 23 Dec 2015 00:36:51 +0100
Subject: [PATCH] MINOR: ironbee: Add Ironbee WAF engine

This patch adds ironbee waf engine:
---
 Makefile      |   8 +-
 src/ironbee.c | 349 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 356 insertions(+), 1 deletion(-)
 create mode 100644 src/ironbee.c

diff --git a/Makefile b/Makefile
index d42d5fb..1ec9071 100644
--- a/Makefile
+++ b/Makefile
@@ -39,6 +39,7 @@
 #   USE_DL               : enable it if your system requires -ldl. Automatic on Linux.
 #   USE_DEVICEATLAS      : enable DeviceAtlas api.
 #   USE_51DEGREES        : enable third party device detection library from 51Degrees
+#   USE_IRONBEE          : enable the ironbee waf
 #
 # Options can be forced by specifying "USE_xxx=1" or can be disabled by using
 # "USE_xxx=" (empty string).
@@ -606,6 +607,11 @@ $(error unable to automatically detect the Lua library name, you can enforce its
 endif
 endif
 
+ifneq ($(USE_IRONBEE),)
+OPTIONS_LDFLAGS += $(if $(IRONBEE_LIB),-L$(IRONBEE_LIB) -lironbee -libutil)
+OPTIONS_CFLAGS += -DUSE_IRONBEE $(if $(IRONBEE_INC),-I$(IRONBEE_INC))
+endif
+
 OPTIONS_LDFLAGS += $(LUA_LD_FLAGS) -l$(LUA_LIB_NAME) -lm
 ifneq ($(USE_DL),)
 OPTIONS_LDFLAGS += -ldl
@@ -750,7 +756,7 @@ OBJS = src/haproxy.o src/base64.o src/protocol.o \
        src/session.o src/stream.o src/hdr_idx.o src/ev_select.o src/signal.o \
        src/acl.o src/sample.o src/memory.o src/freq_ctr.o src/auth.o src/proto_udp.o \
        src/compression.o src/payload.o src/hash.o src/pattern.o src/map.o \
-       src/namespace.o src/mailers.o src/dns.o src/vars.o
+       src/namespace.o src/mailers.o src/dns.o src/vars.o src/ironbee.o
 
 EBTREE_OBJS = $(EBTREE_DIR)/ebtree.o \
               $(EBTREE_DIR)/eb32tree.o $(EBTREE_DIR)/eb64tree.o \
diff --git a/src/ironbee.c b/src/ironbee.c
new file mode 100644
index 0000000..1f7d56a
--- /dev/null
+++ b/src/ironbee.c
@@ -0,0 +1,349 @@
+#ifdef USE_IRONBEE
+
+#include <ironbee/config.h>       /* For ib_cfgparser_* */
+#include <ironbee/engine.h>       /* For many things */
+#include <ironbee/state_notify.h> /* For ib_state_notify_* */
+#include <ironbee/string.h>       /* For IB_S2SL */
+
+#include <common/standard.h>
+
+#include <types/proto_http.h>
+#include <types/sample.h>
+#include <types/stream.h>
+
+#include <proto/arg.h>
+#include <proto/connection.h>
+#include <proto/hdr_idx.h>
+#include <proto/obj_type.h>
+#include <proto/proto_http.h>
+#include <proto/sample.h>
+
+struct ironbee_config {
+	char *file;
+	ib_engine_t *engine;
+};
+
+static int analyse(const struct arg *args, struct sample *smp,
+                   const char *kw, void *private)
+{
+	struct ironbee_config *conf;
+	struct connection *cli_conn;
+	char ip_src[INET6_ADDRSTRLEN + 1];
+	char ip_dst[INET6_ADDRSTRLEN + 1];
+	const char *ret;
+	struct http_txn *txn;
+	struct http_msg *msg;
+	int old_idx, cur_idx;
+	const char *cur_ptr, *cur_next, *p;
+	struct hdr_idx_elem *cur_hdr;
+	const char *hn, *hv;
+	int hnl, hvl;
+	ib_conn_t *conn;
+	ib_tx_t *tx;
+	ib_status_t rc;
+	ib_parsed_req_line_t *req_line;
+	ib_parsed_headers_t *headers;
+	int ret_code;
+
+	conf = (struct ironbee_config *)args[0].data.usr;
+	txn = smp->strm->txn;
+	ret_code = 0;
+
+	/* Create Connection
+	 *
+	 * A connection is some TCP/IP information and a sequence of transactions.
+	 * Its primary purpose is to associate transactions.
+	 */
+	rc = ib_conn_create(conf->engine, &conn, NULL);
+	if (rc != IB_OK)
+		goto analyse_return;
+
+	/* Extract source IP informations. */
+
+	cli_conn = objt_conn(smp->sess->origin);
+	if (!cli_conn)
+		goto analyse_destroy_conn;
+
+	conn_get_to_addr(cli_conn);
+
+	switch (cli_conn->addr.from.ss_family) {
+	case AF_INET:
+		ret = inet_ntop(AF_INET, &((struct sockaddr_in *)&cli_conn->addr.from)->sin_addr,
+		                ip_src, INET6_ADDRSTRLEN + 1);
+		break;
+	case AF_INET6:
+		ret = inet_ntop(AF_INET6, &((struct sockaddr_in6 *)&cli_conn->addr.from)->sin6_addr,
+		                ip_src, INET6_ADDRSTRLEN + 1);
+		break;
+	default:
+		goto analyse_destroy_conn;
+	}
+	if (!ret)
+		goto analyse_destroy_conn;
+
+	switch (cli_conn->addr.to.ss_family) {
+	case AF_INET:
+		ret = inet_ntop(AF_INET, &((struct sockaddr_in *)&cli_conn->addr.to)->sin_addr,
+		                ip_dst, INET6_ADDRSTRLEN + 1);
+		break;
+	case AF_INET6:
+		ret = inet_ntop(AF_INET6, &((struct sockaddr_in6 *)&cli_conn->addr.to)->sin6_addr,
+		                ip_dst, INET6_ADDRSTRLEN + 1);
+		break;
+	default:
+		goto analyse_destroy_conn;
+	}
+	if (!ret)
+		goto analyse_destroy_conn;
+
+	conn->local_ipstr  = ip_dst;
+	conn->local_port   = get_host_port(&cli_conn->addr.to);
+	conn->remote_ipstr = ip_src;
+	conn->remote_port  = get_host_port(&cli_conn->addr.from);
+
+	/* Connection Opened */
+	rc = ib_state_notify_conn_opened(conf->engine, conn);
+	if (rc != IB_OK)
+		goto analyse_destroy_conn;
+
+	/* Create Transaction */
+	rc = ib_tx_create(&tx, conn, NULL);
+	if (rc != IB_OK)
+		goto analyse_destroy_conn;
+
+	/* Request Started */
+	rc = ib_parsed_req_line_create(&req_line, tx->mm,
+	                     /* raw */ txn->req.chn->buf->p,                    txn->req.sl.rq.l,
+	                  /* method */ txn->req.chn->buf->p,                    txn->req.sl.rq.m_l,
+	                     /* uri */ txn->req.chn->buf->p + txn->req.sl.rq.u, txn->req.sl.rq.u_l,
+	                /* protocol */ txn->req.chn->buf->p + txn->req.sl.rq.v, txn->req.sl.rq.v_l);
+	if (rc != IB_OK)
+		goto analyse_destroy_tx;
+
+	rc = ib_state_notify_request_started(conf->engine, tx, req_line);
+	if (rc != IB_OK)
+		return 0;
+
+	/* Request Header */
+	rc = ib_parsed_headers_create(&headers, tx->mm);
+	if (rc != IB_OK)
+		goto analyse_destroy_tx;
+
+	/* Build array of headers. */
+	old_idx = 0;
+	cur_next = txn->req.chn->buf->p + hdr_idx_first_pos(&smp->strm->txn->hdr_idx);
+   while (1) {
+		cur_idx = txn->hdr_idx.v[old_idx].next;
+		if (!cur_idx)
+			break;
+		old_idx = cur_idx;
+
+		cur_hdr  = &txn->hdr_idx.v[cur_idx];
+		cur_ptr  = cur_next;
+		cur_next = cur_ptr + cur_hdr->len + cur_hdr->cr + 1; 
+
+		/* look for ': *'. */
+		hn = cur_ptr;
+		for (p = cur_ptr; p < cur_ptr + cur_hdr->len && *p != ':'; p++);
+		if (p >= cur_ptr + cur_hdr->len)
+			continue;
+		hnl = p - hn;
+		p++; 
+		while (p < cur_ptr + cur_hdr->len && ( *p == ' ' || *p == '\t' ))
+			p++;
+		if (p >= cur_ptr + cur_hdr->len)
+			continue;
+		hv = p;
+		hvl = cur_ptr + cur_hdr->len-p;
+
+		/* Add header. */
+		rc = ib_parsed_headers_add(headers, hn, hnl, hv, hvl);
+		if (rc != IB_OK)
+			goto analyse_destroy_tx;
+	}
+
+	rc = ib_state_notify_request_header_data(conf->engine, tx, headers);
+	if (rc != IB_OK)
+		goto analyse_destroy_tx;
+
+	/* Request Header Finished */
+	rc = ib_state_notify_request_header_finished(conf->engine, tx);
+	if (rc != IB_OK)
+		goto analyse_destroy_tx;
+
+	/* Request Body */
+	msg = &txn->req;
+	rc = ib_state_notify_request_body_data(conf->engine, tx,
+	                                       b_ptr(msg->chn->buf, -http_data_rewind(msg)),
+	                                       http_body_bytes(msg));
+	if (rc != IB_OK)
+		goto analyse_destroy_tx;
+
+	/* Request Finished */
+	rc = ib_state_notify_request_finished(conf->engine, tx);
+	if (rc != IB_OK)
+		goto analyse_destroy_tx;
+
+	/* Return ok. */
+	ret_code = 1;
+
+analyse_destroy_tx:
+
+	/* Transaction Done */
+	ib_tx_destroy(tx);
+
+analyse_destroy_conn:
+
+	/* Connection Done */
+	ib_conn_destroy(conn);
+
+analyse_return:
+
+	return ret_code;
+}
+
+/* IronBee requests that server modify headers before further processing. */
+static ib_status_t server_header(ib_tx_t *tx, ib_server_direction_t dir,
+                                 ib_server_header_action_t action,
+                                 const char *name, size_t name_length,
+                                 const char *value, size_t value_length,
+                                 void *cbdata)
+{
+	fprintf(stderr, "%s\n", __FUNCTION__);
+	return IB_OK;
+}
+
+/* IronBee requests that server respond with an error status. */
+static ib_status_t server_error(ib_tx_t *tx, int status, void *cbdata)
+{
+	fprintf(stderr, "%s\n", __FUNCTION__);
+	return IB_OK;
+}
+
+/* IronBee requests that server provide a certain header in error response. */
+static ib_status_t server_error_header(ib_tx_t *tx, const char *name,
+                                       size_t name_length, const char *value,
+                                       size_t value_length, void *cbdata)
+{
+	fprintf(stderr, "%s\n", __FUNCTION__);
+	return IB_OK;
+}
+
+/* IronBee requests that server provide a certain body in error response. */
+static ib_status_t server_error_data(ib_tx_t *tx, const char *data,
+                                     size_t data_length, void *cbdata)
+{
+	fprintf(stderr, "%s\n", __FUNCTION__);
+	return IB_OK;
+}
+
+static ib_status_t server_close(ib_conn_t *conn, ib_tx_t *tx, void *cbdata)
+{
+	fprintf(stderr, "%s\n", __FUNCTION__);
+	return IB_OK;
+}
+
+static ib_status_t server_body_edit(ib_tx_t *tx, ib_server_direction_t dir,
+                                    off_t start, size_t bytes, const char *repl,
+                                    size_t repl_len, void *cbdata)
+{
+	fprintf(stderr, "%s\n", __FUNCTION__);
+	return IB_OK;
+}
+
+static ib_status_t server_body_init(ib_tx_t *tx, int flags, void *x)
+{
+	fprintf(stderr, "%s\n", __FUNCTION__);
+	return IB_OK;
+}
+
+static ib_server_t server = {
+	IB_SERVER_HEADER_DEFAULTS,
+	"haproxy",
+	server_header,       NULL,
+	server_error,        NULL,
+	server_error_header, NULL,
+	server_error_data,   NULL,
+	server_close,        NULL,
+	server_body_edit,    NULL,
+	server_body_init,    NULL
+};
+
+static int load_rules(struct arg *arg, char **err_msg)
+{
+	struct ironbee_config *conf;
+	ib_cfgparser_t *parser;
+	ib_status_t rc;
+
+	/* Reserve a lot of memory. */
+	conf = malloc(sizeof(*conf));
+	if (!conf) {
+		memprintf(err_msg, "out fo memory error");
+		return 0;
+	}
+
+	/* Copy original file. */
+	conf->file = arg->data.str.str;
+
+	/* Create Engine */
+	rc = ib_engine_create(&conf->engine, &server);
+	if (rc != IB_OK) {
+		memprintf(err_msg, "error creating Ironbee engine: %s",
+		          ib_status_to_string(rc));
+		return 0;
+	}
+
+	/* Load configuration */
+
+	rc = ib_cfgparser_create(&parser, conf->engine);
+	if (rc != IB_OK) {
+		memprintf(err_msg, "error loading Ironbee rule file '%s': %s",
+		          conf->file, ib_status_to_string(rc));
+		return 0;
+	}
+
+	rc = ib_engine_config_started(conf->engine, parser);
+	if (rc != IB_OK) {
+		memprintf(err_msg, "error loading Ironbee rule file '%s': %s",
+		          conf->file, ib_status_to_string(rc));
+		return 0;
+	}
+
+	rc = ib_cfgparser_parse(parser, conf->file);
+	if (rc != IB_OK) {
+		memprintf(err_msg, "error loading Ironbee rule file '%s': %s",
+		          conf->file, ib_status_to_string(rc));
+		return 0;
+	}
+
+	rc = ib_engine_config_finished(conf->engine);
+	if (rc != IB_OK) {
+		memprintf(err_msg, "error loading Ironbee rule file '%s': %s",
+		          conf->file, ib_status_to_string(rc));
+		return 0;
+	}
+
+	ib_cfgparser_destroy(parser);
+
+	/* Store the configuration. */
+	arg->type = ARGT_USR;
+	arg->data.usr = (struct userlist *)conf;
+	return 1;
+}
+
+static struct sample_fetch_kw_list sample_fetch_keywords = {ILH, {
+	{ "ironbee", analyse, ARG1(1,STR), load_rules, SMP_T_SINT, SMP_USE_HRQHV },
+	{ /* END */ },
+}};
+
+__attribute__((constructor))
+static void __ironbee_init(void)
+{
+	/* Initialize IronBee */
+	ib_initialize();
+
+	/* Register sample fetches keywords. */
+	sample_register_fetches(&sample_fetch_keywords);
+}
+
+#endif
-- 
2.1.4


#2

Hi Thierry,

Maybe you should shre the link to this discource page on the ML as well, to get more feedback about this nice feature!


#3

I find it interesting to see that the analyze portion of the code is only a few tens of lines long. I’ve always read that ironbee was made to be easily portable and I think this is a proof of this statement.

It also shows that the internal API of haproxy 1.6/1.7 is much more suited to run such analysis than what we used to have previously, it may open the way to other future analysis.

I suspect that once we merge Christopher’s filters, the code could be easily ported to use them without requiring any http-request rule and could be a good validation of the model (just for experimental purposes as well).

Regarding the header parser, you could simplify it by calling http_find_full_header2(0, 0, txn->req.chn->buf->p, idx, ctx), it will iteratively return each header with the name in ctx->val, the name length in ctx->del, the start of the header’s value in ctx->val and the value’s length in ctx->vlen. That’s apparently 22 lines which can be removed.

Regarding the accesses to /dev/random, it’s possible that some initialization code needs to be called the first time before the chroot/fork. We’ve seen this in other libs as well, they needed some random and were causing an open of /dev/random or /dev/urandom during the first call.

Out of curiosity, have you checked memory usage and performance impact ?

I think this code could serve as a nice example to show how to contribute code adding support for various libs (device identification, waf, url filters, etc). I’m not sure it would make sense to merge it in mainline considering there is little to no demand for a waf in haproxy (and we’ve even heard a few people congratulate us for not wasting resources on such features), but it depends on what people think about it and how it performs.