As announced by AWS the way buckets are used in the URL is about to change (it is deferrred but it will come) The path style format will disappear https://s3.domain.net/[bucket]/[object] and the only format that will work in the near future will be: https://[bucket].s3.domain.net/[object]
If i put HAProxy in front of S3 storage i will need to forward the [bucket] to the backend or the S3 storage won’t understand how to access that bucket. For understanding creating a rule for each bucket will be no real solution as some users can create new buckets on the fly. So is there a way to capture the first part of the incoming URL (the bucket or subdomain) and send that to the backend?
So if my backend is s3storage.my.lan the storage wants to be connected with https://[bucket].s3storage.my.lan/[object]
I’m not sure I fully understand the issue yet, the subdomain being used by the bucket forms part of the host header and the host header the client used should be passed to the backend unless you are already re-writing it or overriding it in another way…
Maybe you could share your config…?
Here is part of my config that is involved with the part i cannot get working. The total config is way bigger but all the other parts work. The http-request set-uri and the http-request set-method i added later but without those it also does not work.
# Automaticaly generated, dont edit manually.
# Generated on: 2021-08-05 10:25
global
maxconn 1000
stats socket /tmp/haproxy.socket level admin expose-fd listeners
uid 80
gid 80
nbproc 1
nbthread 1
hard-stop-after 15m
chroot /tmp/haproxy_chroot
daemon
server-state-file /tmp/haproxy_server_state
# Modern browser compatibility only as mentioned here:
# https://wiki.mozilla.org/Security/Server_Side_TLS
# ssl-default-bind-ciphers ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!3DES:!MD5:!PSK
ssl-default-bind-ciphers ECDHE-RSA-AES256-GCM-SHA384:!ECDHE-RSA-AES128-GCM-SHA256:!ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:!DHE-RSA-AES128-GCM-SHA256:!DHE-DSS-AES128-GCM-SHA256:!kEDH+AESGCM:!ECDHE-RSA-AES128-SHA256:!ECDHE-ECDSA-AES128-SHA256:!ECDHE-RSA-AES128-SHA:!ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:!DHE-RSA-AES128-SHA256:!DHE-RSA-AES128-SHA:!DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!3DES:!MD5:!PSK
ssl-default-bind-options no-sslv3 no-tlsv10 no-tlsv11
ssl-default-server-ciphers ECDHE-RSA-AES256-GCM-SHA384:!ECDHE-RSA-AES128-GCM-SHA256:!ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:!DHE-RSA-AES128-GCM-SHA256:!DHE-DSS-AES128-GCM-SHA256:!kEDH+AESGCM:!ECDHE-RSA-AES128-SHA256:!ECDHE-ECDSA-AES128-SHA256:!ECDHE-RSA-AES128-SHA:!ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:!DHE-RSA-AES128-SHA256:!DHE-RSA-AES128-SHA:!DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!3DES:!MD5:!PSK
ssl-default-server-options no-sslv3 no-tlsv10 no-tlsv11
tune.ssl.default-dh-param 4096
# Time-to-first-Byte (TTFB) value needs to be optimized based on
# the actual public certificate chain see
# https://www.igvita.com/2013/10/24
# /optimizing-tls-record-size-and-buffering-latency/
tune.ssl.maxrecord 1370
listen HAProxyLocalStats
bind 127.0.0.1:2200 name localstats
mode http
stats enable
stats admin if TRUE
stats show-legends
stats uri /haproxy/haproxy_stats.php?haproxystats=1
timeout client 5000
timeout connect 5000
timeout server 5000
frontend WAN_443-merged
bind 192.168.1.10:443 name 192.168.1.10:443
mode tcp
log global
timeout client 7200000
tcp-request inspect-delay 5s
tcp-request content accept if { req.ssl_hello_type 1 } or !{ req.ssl_hello_type 1 }
acl https1 req.ssl_hello_type 1
acl https2 req.ssl_sni -m end -i vpn.playhere.nl
acl https3 req.ssl_sni -m end -i ssh.playhere.nl
acl https4 req.ssl_sni -m end -i .playhere.nl
use_backend WAN_HTTPS_ipvANY if https1 !https2 !https3 https4
default_backend CloudflareSinkhole_ipvANY
frontend WAN_HTTPS-merged
bind 127.0.0.1:2043 name 127.0.0.1:2043 no-sslv3 no-tlsv10 no-tlsv11 ssl crt-list /var/etc/haproxy/WAN_HTTPS.crt_list accept-proxy npn http/1.1
mode http
log global
option http-keep-alive
option forwardfor
acl https ssl_fc
http-request set-header X-Forwarded-Proto http if !https
http-request set-header X-Forwarded-Proto https if https
timeout client 7200000
# Remove headers that expose security-sensitive information.
rspidel ^Server:.*$
rspidel ^X-Powered-By:.*$
rspidel ^X-AspNet-Version:.*$
rspadd Strict-Transport-Security:\ max-age=31536000;\ includeSubDomains
acl aclcrt_WAN_HTTPS var(txn.txnhost) -m reg -i ^s3\.playhere\.nl(:([0-9]){1,5})?$
acl s3 var(txn.txnhost) -m end -i s3.playhere.nl
http-request set-var(txn.txnhost) hdr(host)
use_backend S3_ipvANY if s3 aclcrt_WAN_HTTPS_s3
backend WAN_HTTPS_ipvANY
mode tcp
id 130
log global
timeout connect 30000
timeout server 7200000
retries 3
server WAN_HHTPS 127.0.0.1:2043 id 131 send-proxy
backend S3_ipvANY
mode http
id 139
log global
timeout connect 30000
timeout server 30000
retries 3
http-request set-uri https://%[req.hdr(Host)]%[path]?%[query]
http-request set-method GET
server S3 192.168.111.8:9000 id 123 check inter 1000 weight 1