netty: Memory leak when creating ReferenceCountedSslServerContext instances

We’re currently running a 4.1.7.Final-SNAPSHOT version of Netty that pre-dates the official 4.1.7.Final release (our SNAPSHOT is roughly from January 6, 2017).

We today attempted a release with 4.1.8.Final-SNAPSHOT and I’ve tried 4.1.7.Final as well. There is suddenly a large number of the following (new) Exceptions and the server runs out of memory quickly until Linux’s oom_killer steps in and axes the process. There are no Java OOMEs nor do YourKit memory snapshots show anything out of the ordinary which points towards direct memory being leaked.

Looking the recent commit history there appears to be this dd055c01c78c3a6ce5b9195890486a6e1ab072d2 change that is potentially related (happened after January 6 but before 4.1.7.Final release).

javax.net.ssl.SSLException: error:140D00CF:SSL routines:SSL_write:protocol is shutdown
	at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.shutdownWithError(ReferenceCountedOpenSslEngine.java:719) ~[netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.shutdownWithError(ReferenceCountedOpenSslEngine.java:708) ~[netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.wrap(ReferenceCountedOpenSslEngine.java:685) ~[netty-all-4.1.7.Final.jar:4.1.7.Final]
	at javax.net.ssl.SSLEngine.wrap(SSLEngine.java:509) ~[?:1.8.0_102]
	at io.netty.handler.ssl.SslHandler.wrap(SslHandler.java:746) ~[netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.handler.ssl.SslHandler.wrap(SslHandler.java:578) ~[netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.handler.ssl.SslHandler.wrapAndFlush(SslHandler.java:550) ~[netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.handler.ssl.SslHandler.flush(SslHandler.java:531) ~[netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:777) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:769) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:750) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.flush(CombinedChannelDuplexHandler.java:530) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.ChannelOutboundHandlerAdapter.flush(ChannelOutboundHandlerAdapter.java:115) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.CombinedChannelDuplexHandler.flush(CombinedChannelDuplexHandler.java:355) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:777) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:769) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:750) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.ChannelDuplexHandler.flush(ChannelDuplexHandler.java:117) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:777) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:769) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:750) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.ChannelDuplexHandler.flush(ChannelDuplexHandler.java:117) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:777) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:769) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:750) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.ChannelOutboundHandlerAdapter.flush(ChannelOutboundHandlerAdapter.java:115) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:777) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:769) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:750) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.ChannelOutboundHandlerAdapter.flush(ChannelOutboundHandlerAdapter.java:115) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:777) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:769) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:750) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.ChannelOutboundHandlerAdapter.flush(ChannelOutboundHandlerAdapter.java:115) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:777) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeFlush(AbstractChannelHandlerContext.java:769) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.AbstractChannelHandlerContext.flush(AbstractChannelHandlerContext.java:750) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.ChannelDuplexHandler.flush(ChannelDuplexHandler.java:117) [netty-all-4.1.7.Final.jar:4.1.7.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeFlush0(AbstractChannelHandlerContext.java:777) [netty-all-4.1.7.Final.jar:4.1.7.Final]
...