traefik: Internal Server Error on custom error page

Might be related to https://github.com/containous/traefik/issues/3643.

Do you want to request a feature or report a bug?

Bug

What did you do?

Setup a custom html page and redirect 50X following this example (https://docs.traefik.io/configuration/commons/#custom-error-pages)

Thing to note:

  • We are running traefik behind an AWS ALB
  • The ALB is responsible for HTTPS and only speak HTTP 1.1 to traefik
  • The error page is hosted on the same machine as traefik and distributed through a nginx on localhost:8083

What did you expect to see?

The custom error page when our server answers a 50X error.

What did you see instead?

Internal Server Error

Output of traefik version: (What version of Traefik are you using?)

Version:      v1.6.5
Codename:     tetedemoine
Go version:   go1.10.3
Built:        2018-07-10_03:52:30PM
OS/Arch:      linux/amd64

What is your environment & configuration (arguments, toml, provider, platform, …)?

configuration json 
{
 "LifeCycle": {
  "RequestAcceptGraceTimeout": 0,
  "GraceTimeOut": 0
 },
 "GraceTimeOut": 0,
 "Debug": false,
 "CheckNewVersion": true,
 "SendAnonymousUsage": false,
 "AccessLogsFile": "",
 "AccessLog": {
  "file": "/var/log/traefik/access.log",
  "format": "common"
 },
 "TraefikLogsFile": "",
 "TraefikLog": {
  "file": "xxxx",
  "format": "xxxx"
 },
 "Tracing": null,
 "LogLevel": "DEBUG",
 "EntryPoints": {
  "http": {
   "Address": "xxxx",
   "TLS": null,
   "Redirect": null,
   "Auth": null,
   "WhitelistSourceRange": null,
   "WhiteList": null,
   "Compress": false,
   "ProxyProtocol": null,
   "ForwardedHeaders": {
    "Insecure": false,
    "TrustedIPs": []
   }
  },
  "profiler": {
   "Address": "xxxx",
   "TLS": null,
   "Redirect": null,
   "Auth": null,
   "WhitelistSourceRange": null,
   "WhiteList": {
    "useXForwardedFor": true
   },
   "Compress": false,
   "ProxyProtocol": null,
   "ForwardedHeaders": {
    "Insecure": false,
    "TrustedIPs": []
   }
  },
  "traefik": {
   "Address": "xxxx",
   "TLS": null,
   "Redirect": null,
   "Auth": null,
   "WhitelistSourceRange": null,
   "WhiteList": {
    "useXForwardedFor": true
   },
   "Compress": false,
   "ProxyProtocol": null,
   "ForwardedHeaders": {
    "Insecure": false,
    "TrustedIPs": []
   }
  }
 },
 "Cluster": null,
 "Constraints": [],
 "ACME": null,
 "DefaultEntryPoints": [
  "http"
 ],
 "ProvidersThrottleDuration": 2000000000,
 "MaxIdleConnsPerHost": 200,
 "IdleTimeout": 0,
 "InsecureSkipVerify": false,
 "RootCAs": null,
 "Retry": null,
 "HealthCheck": {
  "Interval": 30000000000
 },
 "RespondingTimeouts": {
  "ReadTimeout": 900000000000,
  "WriteTimeout": 900000000000,
  "IdleTimeout": 900000000000
 },
 "ForwardingTimeouts": {
  "DialTimeout": 900000000000,
  "ResponseHeaderTimeout": 900000000000
 },
 "AllowMinWeightZero": false,
 "Web": null,
 "Docker": null,
 "File": {
  "Watch": true,
  "Filename": "",
  "Constraints": null,
  "Trace": false,
  "TemplateVersion": 0,
  "DebugLogGeneratedTemplate": false,
  "Directory": "",
  "TraefikFile": ""
 },
 "Marathon": null,
 "Consul": null,
 "ConsulCatalog": null,
 "Etcd": null,
 "Zookeeper": null,
 "Boltdb": null,
 "Kubernetes": null,
 "Mesos": null,
 "Eureka": null,
 "ECS": null,
 "Rancher": null,
 "DynamoDB": null,
 "ServiceFabric": null,
 "Rest": null,
 "API": {
  "EntryPoint": "traefik",
  "Dashboard": true,
  "Debug": false,
  "CurrentConfigurations": null,
  "Statistics": null
 },
 "Metrics": {
  "Prometheus": null,
  "Datadog": null,
  "StatsD": {
   "Address": "xxxx",
   "PushInterval": "10s"
  },
  "InfluxDB": null
 },
 "Ping": {
  "EntryPoint": "http"
 },
 "ConfigFile": "/etc/traefik/traefik.toml"
}

The configuration of our frontend with the custom error page:


[backends]
  # Nasty duplication because frontend reusing the same backend is not working
  [backends.ark_front]
    [backends.ark_front.servers.one]
      url = "http://xx"
  [backends.ark_back]
    [backends.ark_back.servers.one]
      url = "http://xx"
  [backends.profiler]
    [backends.profiler.servers.one]
      url = "http://xx"
  [backends.preview]
    [backends.preview.servers.one]
      url = "http://localhost:8183"
  [backends.down]
    [backends.down.servers.one]
      url = "http://localhost:8184"

[frontends]

  [frontends.ark_front]
    backend = "ark_front"
    [frontends.ark_front.headers]
      SSLRedirect = true
      [frontends.ark_front.headers.SSLProxyHeaders]
        X-Forwarded-Proto = "https"
    [frontends.ark_front.errors]
      [frontends.ark_front.errors.network]
      status = ["500-599"]
      backend = "down"
      query = "/down.html"

If applicable, please paste the log output in DEBUG level (--logLevel=DEBUG switch)

time="2018-07-26T09:34:53Z" level=debug msg="vulcand/oxy/roundrobin/rr: begin ServeHttp on request" Request="{\"Method\":\"GET\",\"URL\":{\"Scheme\":\"\",\"Opaque\":\"\",\"User\":null,\"Host\":\"\",\"Path\":\"/helicopter/5b589eb656039a08d52e3ca4\",\"RawPath\":\"\",\"ForceQuery\":false,\"RawQuery\":\"\",\"Fragment\":\"\"},\"Proto\":\"HTTP/1.1\",\"ProtoMajor\":1,\"ProtoMinor\":1,\"Header\":{\"Accept\":[\"*/*\"],\"User-Agent\":[\"curl/7.54.0\"],\"X-Amzn-Trace-Id\":[\"Root=1-5b5995bd-24de8cb3a221773e3f601b8d\"],\"X-Forwarded-For\":[\"83.167.33.114\"],\"X-Forwarded-Port\":[\"443\"],\"X-Forwarded-Proto\":[\"https\"]},\"ContentLength\":0,\"TransferEncoding\":null,\"Host\":\"<REDACTED>\",\"Form\":null,\"PostForm\":null,\"MultipartForm\":null,\"Trailer\":null,\"RemoteAddr\":\"<REDACTED>:6174\",\"RequestURI\":\"/helicopter/5b589eb656039a08d52e3ca4\",\"TLS\":null}"
time="2018-07-26T09:34:53Z" level=debug msg="vulcand/oxy/roundrobin/rr: Forwarding this request to URL" Request="{\"Method\":\"GET\",\"URL\":{\"Scheme\":\"\",\"Opaque\":\"\",\"User\":null,\"Host\":\"\",\"Path\":\"/helicopter/5b589eb656039a08d52e3ca4\",\"RawPath\":\"\",\"ForceQuery\":false,\"RawQuery\":\"\",\"Fragment\":\"\"},\"Proto\":\"HTTP/1.1\",\"ProtoMajor\":1,\"ProtoMinor\":1,\"Header\":{\"Accept\":[\"*/*\"],\"User-Agent\":[\"curl/7.54.0\"],\"X-Amzn-Trace-Id\":[\"Root=1-5b5995bd-24de8cb3a221773e3f601b8d\"],\"X-Forwarded-For\":[\"83.167.33.114\"],\"X-Forwarded-Port\":[\"443\"],\"X-Forwarded-Proto\":[\"https\"]},\"ContentLength\":0,\"TransferEncoding\":null,\"Host\":\"<REDACTED>\",\"Form\":null,\"PostForm\":null,\"MultipartForm\":null,\"Trailer\":null,\"RemoteAddr\":\"<REDACTED>:6174\",\"RequestURI\":\"/helicopter/5b589eb656039a08d52e3ca4\",\"TLS\":null}" ForwardURL="http://app.local.<REDACTED>"
time="2018-07-26T09:34:53Z" level=debug msg="vulcand/oxy/forward: begin ServeHttp on request" Request="{\"Method\":\"GET\",\"URL\":{\"Scheme\":\"http\",\"Opaque\":\"\",\"User\":null,\"Host\":\"app.local.<REDACTED>\",\"Path\":\"\",\"RawPath\":\"\",\"ForceQuery\":false,\"RawQuery\":\"\",\"Fragment\":\"\"},\"Proto\":\"HTTP/1.1\",\"ProtoMajor\":1,\"ProtoMinor\":1,\"Header\":{\"Accept\":[\"*/*\"],\"User-Agent\":[\"curl/7.54.0\"],\"X-Amzn-Trace-Id\":[\"Root=1-5b5995bd-24de8cb3a221773e3f601b8d\"],\"X-Forwarded-For\":[\"83.167.33.114\"],\"X-Forwarded-Port\":[\"443\"],\"X-Forwarded-Proto\":[\"https\"]},\"ContentLength\":0,\"TransferEncoding\":null,\"Host\":\"<REDACTED>\",\"Form\":null,\"PostForm\":null,\"MultipartForm\":null,\"Trailer\":null,\"RemoteAddr\":\"<REDACTED>:6174\",\"RequestURI\":\"/helicopter/5b589eb656039a08d52e3ca4\",\"TLS\":null}"
time="2018-07-26T09:34:53Z" level=debug msg="vulcand/oxy/forward/http: begin ServeHttp on request" Request="{\"Method\":\"GET\",\"URL\":{\"Scheme\":\"http\",\"Opaque\":\"\",\"User\":null,\"Host\":\"app.local.<REDACTED>\",\"Path\":\"\",\"RawPath\":\"\",\"ForceQuery\":false,\"RawQuery\":\"\",\"Fragment\":\"\"},\"Proto\":\"HTTP/1.1\",\"ProtoMajor\":1,\"ProtoMinor\":1,\"Header\":{\"Accept\":[\"*/*\"],\"User-Agent\":[\"curl/7.54.0\"],\"X-Amzn-Trace-Id\":[\"Root=1-5b5995bd-24de8cb3a221773e3f601b8d\"],\"X-Forwarded-For\":[\"83.167.33.114\"],\"X-Forwarded-Port\":[\"443\"],\"X-Forwarded-Proto\":[\"https\"]},\"ContentLength\":0,\"TransferEncoding\":null,\"Host\":\"<REDACTED>\",\"Form\":null,\"PostForm\":null,\"MultipartForm\":null,\"Trailer\":null,\"RemoteAddr\":\"<REDACTED>:6174\",\"RequestURI\":\"/helicopter/5b589eb656039a08d52e3ca4\",\"TLS\":null}"
time="2018-07-26T09:34:53Z" level=debug msg="Upstream ResponseWriter of type *pipelining.writerWithoutCloseNotify does not implement http.CloseNotifier. Returning dummy channel."
time="2018-07-26T09:34:53Z" level=debug msg="vulcand/oxy/forward/http: Round trip: http://app.local.<REDACTED>, code: 502, Length: 138, duration: 3.16182ms"
time="2018-07-26T09:34:53Z" level=debug msg="vulcand/oxy/forward/http: completed ServeHttp on request" Request="{\"Method\":\"GET\",\"URL\":{\"Scheme\":\"http\",\"Opaque\":\"\",\"User\":null,\"Host\":\"app.local.<REDACTED>\",\"Path\":\"\",\"RawPath\":\"\",\"ForceQuery\":false,\"RawQuery\":\"\",\"Fragment\":\"\"},\"Proto\":\"HTTP/1.1\",\"ProtoMajor\":1,\"ProtoMinor\":1,\"Header\":{\"Accept\":[\"*/*\"],\"User-Agent\":[\"curl/7.54.0\"],\"X-Amzn-Trace-Id\":[\"Root=1-5b5995bd-24de8cb3a221773e3f601b8d\"],\"X-Forwarded-For\":[\"83.167.33.114\"],\"X-Forwarded-Port\":[\"443\"],\"X-Forwarded-Proto\":[\"https\"]},\"ContentLength\":0,\"TransferEncoding\":null,\"Host\":\"<REDACTED>\",\"Form\":null,\"PostForm\":null,\"MultipartForm\":null,\"Trailer\":null,\"RemoteAddr\":\"<REDACTED>:6174\",\"RequestURI\":\"/helicopter/5b589eb656039a08d52e3ca4\",\"TLS\":null}"
time="2018-07-26T09:34:53Z" level=debug msg="vulcand/oxy/forward: completed ServeHttp on request" Request="{\"Method\":\"GET\",\"URL\":{\"Scheme\":\"http\",\"Opaque\":\"\",\"User\":null,\"Host\":\"app.local.<REDACTED>\",\"Path\":\"\",\"RawPath\":\"\",\"ForceQuery\":false,\"RawQuery\":\"\",\"Fragment\":\"\"},\"Proto\":\"HTTP/1.1\",\"ProtoMajor\":1,\"ProtoMinor\":1,\"Header\":{\"Accept\":[\"*/*\"],\"User-Agent\":[\"curl/7.54.0\"],\"X-Amzn-Trace-Id\":[\"Root=1-5b5995bd-24de8cb3a221773e3f601b8d\"],\"X-Forwarded-For\":[\"83.167.33.114\"],\"X-Forwarded-Port\":[\"443\"],\"X-Forwarded-Proto\":[\"https\"]},\"ContentLength\":0,\"TransferEncoding\":null,\"Host\":\"<REDACTED>\",\"Form\":null,\"PostForm\":null,\"MultipartForm\":null,\"Trailer\":null,\"RemoteAddr\":\"<REDACTED>:6174\",\"RequestURI\":\"/helicopter/5b589eb656039a08d52e3ca4\",\"TLS\":null}"
time="2018-07-26T09:34:53Z" level=debug msg="vulcand/oxy/roundrobin/rr: completed ServeHttp on request" Request="{\"Method\":\"GET\",\"URL\":{\"Scheme\":\"\",\"Opaque\":\"\",\"User\":null,\"Host\":\"\",\"Path\":\"/helicopter/5b589eb656039a08d52e3ca4\",\"RawPath\":\"\",\"ForceQuery\":false,\"RawQuery\":\"\",\"Fragment\":\"\"},\"Proto\":\"HTTP/1.1\",\"ProtoMajor\":1,\"ProtoMinor\":1,\"Header\":{\"Accept\":[\"*/*\"],\"User-Agent\":[\"curl/7.54.0\"],\"X-Amzn-Trace-Id\":[\"Root=1-5b5995bd-24de8cb3a221773e3f601b8d\"],\"X-Forwarded-For\":[\"83.167.33.114\"],\"X-Forwarded-Port\":[\"443\"],\"X-Forwarded-Proto\":[\"https\"]},\"ContentLength\":0,\"TransferEncoding\":null,\"Host\":\"<REDACTED>\",\"Form\":null,\"PostForm\":null,\"MultipartForm\":null,\"Trailer\":null,\"RemoteAddr\":\"<REDACTED>:6174\",\"RequestURI\":\"/helicopter/5b589eb656039a08d52e3ca4\",\"TLS\":null}"
time="2018-07-26T09:34:53Z" level=error msg="Caught HTTP Status Code 502, returning error page"
time="2018-07-26T09:34:53Z" level=debug msg="vulcand/oxy/forward: begin ServeHttp on request" Request="{\"Method\":\"GET\",\"URL\":{\"Scheme\":\"\",\"Opaque\":\"\",\"User\":null,\"Host\":\"\",\"Path\":\"/down.html\",\"RawPath\":\"\",\"ForceQuery\":false,\"RawQuery\":\"\",\"Fragment\":\"\"},\"Proto\":\"HTTP/1.1\",\"ProtoMajor\":1,\"ProtoMinor\":1,\"Header\":{\"Accept\":[\"*/*\"],\"User-Agent\":[\"curl/7.54.0\"],\"X-Amzn-Trace-Id\":[\"Root=1-5b5995bd-24de8cb3a221773e3f601b8d\"],\"X-Forwarded-For\":[\"83.167.33.114\"],\"X-Forwarded-Port\":[\"443\"],\"X-Forwarded-Proto\":[\"https\"]},\"ContentLength\":0,\"TransferEncoding\":null,\"Host\":\"\",\"Form\":null,\"PostForm\":null,\"MultipartForm\":null,\"Trailer\":null,\"RemoteAddr\":\"\",\"RequestURI\":\"/down.html\",\"TLS\":null}"
time="2018-07-26T09:34:53Z" level=debug msg="vulcand/oxy/forward/http: begin ServeHttp on request" Request="{\"Method\":\"GET\",\"URL\":{\"Scheme\":\"\",\"Opaque\":\"\",\"User\":null,\"Host\":\"\",\"Path\":\"/down.html\",\"RawPath\":\"\",\"ForceQuery\":false,\"RawQuery\":\"\",\"Fragment\":\"\"},\"Proto\":\"HTTP/1.1\",\"ProtoMajor\":1,\"ProtoMinor\":1,\"Header\":{\"Accept\":[\"*/*\"],\"User-Agent\":[\"curl/7.54.0\"],\"X-Amzn-Trace-Id\":[\"Root=1-5b5995bd-24de8cb3a221773e3f601b8d\"],\"X-Forwarded-For\":[\"83.167.33.114\"],\"X-Forwarded-Port\":[\"443\"],\"X-Forwarded-Proto\":[\"https\"]},\"ContentLength\":0,\"TransferEncoding\":null,\"Host\":\"\",\"Form\":null,\"PostForm\":null,\"MultipartForm\":null,\"Trailer\":null,\"RemoteAddr\":\"\",\"RequestURI\":\"/down.html\",\"TLS\":null}"
time="2018-07-26T09:34:53Z" level=debug msg="vulcand/oxy/forward/http: Round trip: /down.html, code: 500, Length: 21, duration: 42.888µs"
time="2018-07-26T09:34:53Z" level=debug msg="vulcand/oxy/forward/http: completed ServeHttp on request" Request="{\"Method\":\"GET\",\"URL\":{\"Scheme\":\"\",\"Opaque\":\"\",\"User\":null,\"Host\":\"\",\"Path\":\"/down.html\",\"RawPath\":\"\",\"ForceQuery\":false,\"RawQuery\":\"\",\"Fragment\":\"\"},\"Proto\":\"HTTP/1.1\",\"ProtoMajor\":1,\"ProtoMinor\":1,\"Header\":{\"Accept\":[\"*/*\"],\"User-Agent\":[\"curl/7.54.0\"],\"X-Amzn-Trace-Id\":[\"Root=1-5b5995bd-24de8cb3a221773e3f601b8d\"],\"X-Forwarded-For\":[\"83.167.33.114\"],\"X-Forwarded-Port\":[\"443\"],\"X-Forwarded-Proto\":[\"https\"]},\"ContentLength\":0,\"TransferEncoding\":null,\"Host\":\"\",\"Form\":null,\"PostForm\":null,\"MultipartForm\":null,\"Trailer\":null,\"RemoteAddr\":\"\",\"RequestURI\":\"/down.html\",\"TLS\":null}"
time="2018-07-26T09:34:53Z" level=debug msg="vulcand/oxy/forward: completed ServeHttp on request" Request="{\"Method\":\"GET\",\"URL\":{\"Scheme\":\"\",\"Opaque\":\"\",\"User\":null,\"Host\":\"\",\"Path\":\"/down.html\",\"RawPath\":\"\",\"ForceQuery\":false,\"RawQuery\":\"\",\"Fragment\":\"\"},\"Proto\":\"HTTP/1.1\",\"ProtoMajor\":1,\"ProtoMinor\":1,\"Header\":{\"Accept\":[\"*/*\"],\"User-Agent\":[\"curl/7.54.0\"],\"X-Amzn-Trace-Id\":[\"Root=1-5b5995bd-24de8cb3a221773e3f601b8d\"],\"X-Forwarded-For\":[\"83.167.33.114\"],\"X-Forwarded-Port\":[\"443\"],\"X-Forwarded-Proto\":[\"https\"]},\"ContentLength\":0,\"TransferEncoding\":null,\"Host\":\"\",\"Form\":null,\"PostForm\":null,\"MultipartForm\":null,\"Trailer\":null,\"RemoteAddr\":\"\",\"RequestURI\":\"/down.html\",\"TLS\":null}"

From this log I understand that the “down” backend answers a 500 to the request. The thing is, I have no log on the nginx (even in debug mode).

What I get when doing the curl manually to the error page endpoint:

$ curl -vvv http://localhost:8184/down.html
*   Trying 127.0.0.1...
* Connected to localhost (127.0.0.1) port 8184 (#0)
> GET /down.html HTTP/1.1
> Host: localhost:8184
> User-Agent: curl/7.47.0
> Accept: */*
>
< HTTP/1.1 200 OK
< Server: nginx/1.10.3 (Ubuntu)
< Date: Thu, 26 Jul 2018 09:40:05 GMT
< Content-Type: text/html
< Content-Length: 2022
< Last-Modified: Thu, 26 Jul 2018 08:26:20 GMT
< Connection: keep-alive
< ETag: "5b5985ac-7e6"
< Accept-Ranges: bytes
<
<!DOCTYPE >
<html>
<REDACTED>
</html>

About this issue

  • Original URL
  • State: closed
  • Created 6 years ago
  • Reactions: 4
  • Comments: 22 (10 by maintainers)

Most upvoted comments

@yuklia could you open a new issue with your configuration, labels, etc.

+2
ldez on Dec 3, 2018

Experiencing the same problem Trying to setup a custom error page for 403 to show the login page Also encountering “internal server error” on forward to error page, with the host hosting the error page not showing any request in their log

Using Traefik 1.7.0.RC2

+2
Gibletron on Feb 1, 2019

I can try to dig up an example during the weekend or early next week. I’m on the road right now.

+1
ederion on Feb 1, 2019

Has there been a new issue opened with this? I’m facing the same problem.

+1
vertoforce on Feb 1, 2019

/ # traefik version Version: v1.7.4 Codename: maroilles Go version: go1.11.1 Built: 2018-10-30_10:44:30AM OS/Arch: linux/amd64

+1
yuklia on Dec 3, 2018

I have the same problem. If I run 1.7, I get “Internal server error” instead of the page that my endpoint designated for error is supposed to return.

Running traefik on :latest (currently V1.6.5 / TETEDEMOINE) the issue does not present itself.

+1
signaleleven on Feb 1, 2019
Read more comments on GitHub
← traefik: Internal server error in buffering
traefik: Kubernetes Priority Annotation Doesn't Work →