caddy: Incorrect behavior when encode directive is used
Hello,
I’ve noticed that when we use encode gzip zstd directive, the Accept-Ranges header is removed even when the response is not compressed at all. For example,
Incorrect behavior
Issuing this request i expect the Accept-Ranges header to be present in the response
(user@localhost) ~ http -h 'https://dev.example.com/api/browse/download/media/videos/3.mp4' Authorization:"Bearer testapikey"
# Request headers
GET /api/browse/download/media/videos/3.mp4 HTTP/1.1
Accept: */*
Accept-Encoding: gzip, deflate
Authorization: Bearer testapikey
Connection: keep-alive
Host: dev.example.com
User-Agent: HTTPie/2.6.0
# Response Headers
HTTP/1.1 200 OK
Cache-Control: public, max-age=1694361074
Content-Disposition: inline; filename="3.mp4"
Content-Length: 2659699
Content-Type: video/mp4
Date: Sat, 10 Sep 2022 15:51:14 GMT
Expires: Fri, 07 Jul 2023 15:51:14 GMT
Last-Modified: Fri, 13 Oct 2017 00:31:34 GMT
Pragma: public
Server: Caddy
X-Real-Size: 2659699
X-Request-Id: NflSKPf5MYRY3A-H26yzv
As you can see from the request/response cycle, no Accept-Ranges header is present.
Caddy logs
[
{
"level": "debug",
"ts": 1662825074.4811828,
"logger": "http.handlers.rewrite",
"msg": "rewrote request",
"request": {
"remote_ip": "10.0.0.2",
"remote_port": "59699",
"proto": "HTTP/1.1",
"method": "GET",
"host": "dev.example.com",
"uri": "/api/browse/download/media/videos/3.mp4",
"headers": {
"Accept-Encoding": [
"gzip, deflate"
],
"Accept": [
"*/*"
],
"Connection": [
"keep-alive"
],
"Authorization": [],
"User-Agent": [
"HTTPie/2.6.0"
]
},
"tls": {
"resumed": false,
"version": 772,
"cipher_suite": 4865,
"proto": "http/1.1",
"server_name": "dev.example.com"
}
},
"method": "GET",
"uri": "index.php"
},
{
"level": "debug",
"ts": 1662825074.481457,
"logger": "http.reverse_proxy.transport.fastcgi",
"msg": "roundtrip",
"request": {
"remote_ip": "10.0.0.2",
"remote_port": "59699",
"proto": "HTTP/1.1",
"method": "GET",
"host": "dev.example.com",
"uri": "index.php",
"headers": {
"X-Forwarded-Proto": [
"https"
],
"X-Forwarded-Host": [
"dev.example.com"
],
"User-Agent": [
"HTTPie/2.6.0"
],
"Accept-Encoding": [
"gzip, deflate"
],
"Accept": [
"*/*"
],
"X-Forwarded-For": [
"10.0.0.2"
],
"Authorization": []
},
"tls": {
"resumed": false,
"version": 772,
"cipher_suite": 4865,
"proto": "http/1.1",
"server_name": "dev.example.com"
}
},
"dial": "php_fpm:9000",
"env": {
"SERVER_PROTOCOL": "HTTP/1.1",
"SSL_CIPHER": "TLS_AES_128_GCM_SHA256",
"HTTP_X_FORWARDED_HOST": "dev.example.com",
"HTTP_AUTHORIZATION": "Bearer testapikey",
"REQUEST_SCHEME": "https",
"SCRIPT_NAME": "/index.php",
"HTTPS": "on",
"AUTH_TYPE": "",
"SERVER_NAME": "dev.example.com",
"SCRIPT_FILENAME": "/opt/app/public/index.php",
"HTTP_ACCEPT_ENCODING": "gzip, deflate",
"HTTP_X_FORWARDED_FOR": "10.0.0.2",
"REMOTE_ADDR": "10.0.0.2",
"HTTP_X_FORWARDED_PROTO": "https",
"REQUEST_METHOD": "GET",
"DOCUMENT_ROOT": "/opt/app/public/",
"HTTP_ACCEPT": "*/*",
"HTTP_HOST": "dev.example.com",
"REQUEST_URI": "/api/browse/download/media/videos/3.mp4",
"SSL_PROTOCOL": "TLSv1.3",
"GATEWAY_INTERFACE": "CGI/1.1",
"REMOTE_IDENT": "",
"CONTENT_TYPE": "",
"SERVER_SOFTWARE": "Caddy/v2.5.2",
"DOCUMENT_URI": "index.php",
"X_REQUEST_ID": "NflSKPf5MYRY3A-H26yzv",
"CONTENT_LENGTH": "",
"PATH_INFO": "",
"QUERY_STRING": "",
"REMOTE_PORT": "59699",
"REMOTE_HOST": "10.0.0.2",
"REMOTE_USER": "",
"SERVER_PORT": "443",
"HTTP_USER_AGENT": "HTTPie/2.6.0"
}
},
{
"level": "debug",
"ts": 1662825074.546504,
"logger": "http.handlers.reverse_proxy",
"msg": "upstream roundtrip",
"upstream": "php_fpm:9000",
"duration": 0.065128102,
"request": {
"remote_ip": "10.0.0.2",
"remote_port": "59699",
"proto": "HTTP/1.1",
"method": "GET",
"host": "dev.example.com",
"uri": "index.php",
"headers": {
"Accept": [
"*/*"
],
"X-Forwarded-For": [
"10.0.0.2"
],
"Authorization": [],
"X-Forwarded-Proto": [
"https"
],
"X-Forwarded-Host": [
"dev.example.com"
],
"User-Agent": [
"HTTPie/2.6.0"
],
"Accept-Encoding": [
"gzip, deflate"
]
},
"tls": {
"resumed": false,
"version": 772,
"cipher_suite": 4865,
"proto": "http/1.1",
"server_name": "dev.example.com"
}
},
"headers": {
"Expires": [
"Fri, 07 Jul 2023 15:51:14 GMT"
],
"X-Real-Size": [
"2659699"
],
"Pragma": [
"public"
],
"Content-Length": [
"2659699"
],
"Content-Type": [
"video/mp4"
],
"Last-Modified": [
"Fri, 13 Oct 2017 00:31:34 GMT"
],
"Cache-Control": [
"public, max-age=1694361074"
],
"Content-Disposition": [
"inline; filename=\"3.mp4\""
],
"Accept-Ranges": [
"bytes"
]
},
"status": 200
}
]
Correct behavior
Issuing this request with the header Accept-Encoding set to None the expected Accept-Ranges shows up
(user@localhost) ~ http -h 'https://dev.example.com/api/browse/download/media/videos/3.mp4' Authorization:"Bearer testapikey" Accept-Encoding: None
# Request headers
GET /api/browse/download/media/videos/3.mp4 HTTP/1.1
Accept: */*
Accept-Encoding: None
Authorization: Bearer testapikey
Connection: keep-alive
Host: dev.example.com
User-Agent: HTTPie/2.6.0
# Response Headers
HTTP/1.1 200 OK
Accept-Ranges: bytes
Cache-Control: public, max-age=1694361125
Content-Disposition: inline; filename="3.mp4"
Content-Length: 2659699
Content-Type: video/mp4
Date: Sat, 10 Sep 2022 15:52:05 GMT
Expires: Fri, 07 Jul 2023 15:52:05 GMT
Last-Modified: Fri, 13 Oct 2017 00:31:34 GMT
Pragma: public
Server: Caddy
X-Real-Size: 2659699
X-Request-Id: XAH4-Cyc4F2EhdYRTdVeS
Caddy logs
[
{
"level": "debug",
"ts": 1662825125.099057,
"logger": "http.handlers.rewrite",
"msg": "rewrote request",
"request": {
"remote_ip": "10.0.0.2",
"remote_port": "2335",
"proto": "HTTP/1.1",
"method": "GET",
"host": "dev.example.com",
"uri": "/api/browse/download/media/videos/3.mp4",
"headers": {
"Accept-Encoding": [
"None"
],
"Accept": [
"*/*"
],
"Connection": [
"keep-alive"
],
"Authorization": [],
"User-Agent": [
"HTTPie/2.6.0"
]
},
"tls": {
"resumed": false,
"version": 772,
"cipher_suite": 4865,
"proto": "http/1.1",
"server_name": "dev.example.com"
}
},
"method": "GET",
"uri": "index.php"
},
{
"level": "debug",
"ts": 1662825125.0992594,
"logger": "http.reverse_proxy.transport.fastcgi",
"msg": "roundtrip",
"request": {
"remote_ip": "10.0.0.2",
"remote_port": "2335",
"proto": "HTTP/1.1",
"method": "GET",
"host": "dev.example.com",
"uri": "index.php",
"headers": {
"User-Agent": [
"HTTPie/2.6.0"
],
"X-Forwarded-Proto": [
"https"
],
"X-Forwarded-Host": [
"dev.example.com"
],
"Accept-Encoding": [
"None"
],
"Accept": [
"*/*"
],
"X-Forwarded-For": [
"10.0.0.2"
],
"Authorization": []
},
"tls": {
"resumed": false,
"version": 772,
"cipher_suite": 4865,
"proto": "http/1.1",
"server_name": "dev.example.com"
}
},
"dial": "php_fpm:9000",
"env": {
"PATH_INFO": "",
"REMOTE_ADDR": "10.0.0.2",
"REMOTE_PORT": "2335",
"X_REQUEST_ID": "XAH4-Cyc4F2EhdYRTdVeS",
"HTTP_X_FORWARDED_HOST": "dev.example.com",
"CONTENT_LENGTH": "",
"SERVER_SOFTWARE": "Caddy/v2.5.2",
"HTTPS": "on",
"HTTP_X_FORWARDED_PROTO": "https",
"REQUEST_SCHEME": "https",
"DOCUMENT_URI": "index.php",
"HTTP_HOST": "dev.example.com",
"SCRIPT_NAME": "/index.php",
"SERVER_PORT": "443",
"HTTP_AUTHORIZATION": "Bearer testapikey",
"AUTH_TYPE": "",
"SERVER_PROTOCOL": "HTTP/1.1",
"REQUEST_URI": "/api/browse/download/media/videos/3.mp4",
"SCRIPT_FILENAME": "/opt/app/public/index.php",
"HTTP_ACCEPT_ENCODING": "None",
"REMOTE_HOST": "10.0.0.2",
"SERVER_NAME": "dev.example.com",
"SSL_PROTOCOL": "TLSv1.3",
"SSL_CIPHER": "TLS_AES_128_GCM_SHA256",
"GATEWAY_INTERFACE": "CGI/1.1",
"REMOTE_IDENT": "",
"CONTENT_TYPE": "",
"REQUEST_METHOD": "GET",
"DOCUMENT_ROOT": "/opt/app/public/",
"HTTP_ACCEPT": "*/*",
"HTTP_USER_AGENT": "HTTPie/2.6.0",
"QUERY_STRING": "",
"REMOTE_USER": "",
"HTTP_X_FORWARDED_FOR": "10.0.0.2"
}
},
{
"level": "debug",
"ts": 1662825125.1613717,
"logger": "http.handlers.reverse_proxy",
"msg": "upstream roundtrip",
"upstream": "php_fpm:9000",
"duration": 0.062165963,
"request": {
"remote_ip": "10.0.0.2",
"remote_port": "2335",
"proto": "HTTP/1.1",
"method": "GET",
"host": "dev.example.com",
"uri": "index.php",
"headers": {
"X-Forwarded-Host": [
"dev.example.com"
],
"Accept-Encoding": [
"None"
],
"Accept": [
"*/*"
],
"X-Forwarded-For": [
"10.0.0.2"
],
"Authorization": [],
"User-Agent": [
"HTTPie/2.6.0"
],
"X-Forwarded-Proto": [
"https"
]
},
"tls": {
"resumed": false,
"version": 772,
"cipher_suite": 4865,
"proto": "http/1.1",
"server_name": "dev.example.com"
}
},
"headers": {
"Cache-Control": [
"public, max-age=1694361125"
],
"Accept-Ranges": [
"bytes"
],
"Last-Modified": [
"Fri, 13 Oct 2017 00:31:34 GMT"
],
"Pragma": [
"public"
],
"X-Real-Size": [
"2659699"
],
"Expires": [
"Fri, 07 Jul 2023 15:52:05 GMT"
],
"Content-Length": [
"2659699"
],
"Content-Type": [
"video/mp4"
],
"Content-Disposition": [
"inline; filename=\"3.mp4\""
]
},
"status": 200
}
]
As you can see from both examples, Content-Length and X-Real-Size are exactly the same, which means there are no encoding happening, however Caddy still removes Accept-Ranges header even when no encoding is happening. leading to some clients assuming no resume support for the download. As expected removing the directive encode directive fixes the problem.
Caddy config
https://dev.example.com {
request_id
header * X-Request-Id "{http.request_id}"
encode gzip zstd
route {
root * /opt/app/public
php_fastcgi php_fpm:9000 {
env X_REQUEST_ID "{http.request_id}"
}
file_server
}
}
About this issue
- Original URL
- State: closed
- Created 2 years ago
- Comments: 22 (11 by maintainers)
It’s a indeed a bug, will be fixed soon.