netty: Epoll initialization can deadlock if done simultaneously with SSLContext

Expected behavior

Initializing Epoll concurrently with SSLContext will not deadlock.

Actual behavior

Initializing Epoll concurrently with SSLContext can deadlock. This is very similar to #7458. That fixes a deadlock inside netty_unix_socket.c. The deadlock here is in netty_linux_socket.c.

What seems to happen is the following:

1. Thread 1 loads Netty epoll, which locks Runtime.class.
2. Thread 2 creates an SSLContext, which eventually tries to initialize sun.nio.ch.IOUtil. This holds IOUtil’s class initialization lock.
3. IOUtil/thread 2 then tries to load the JVM “net” and “nio” native libraries, which tries to lock Runtime.class and blocks.
4. Thread 1 then tries to load sun.io.ch.FileChannelImpl, which also tries to load sun.nio.ch.IOUtil. Now the two threads are deadlocked.

https://github.com/netty/netty/blob/bdaa935756c3ae86f8b5353011d9682d5e247dcc/transport-native-epoll/src/main/c/netty_epoll_linuxsocket.c#L777

Steps to reproduce

Try creating an SSLContext on one thread while loading Epoll on another thread.

Minimal yet complete reproducer code (or URL to code)

No code, unfortunately - this was internal.

Stack traces:

"main" #1 prio=5 os_prio=0 cpu=1508.46ms elapsed=261.43s tid=0x000014d2d0040800 nid=0x32e634 waiting for monitor entry  [0x000014d2d7dd6000]
   java.lang.Thread.State: BLOCKED (on object monitor)
	at java.lang.Runtime.loadLibrary0(java.base@11.0.6/Runtime.java:821)
	- waiting to lock <0x00000007b88cb060> (a java.lang.Runtime)
	at java.lang.System.loadLibrary(java.base@11.0.6/System.java:1870)
	at sun.nio.ch.IOUtil$1.run(java.base@11.0.6/IOUtil.java:438)
	at sun.nio.ch.IOUtil$1.run(java.base@11.0.6/IOUtil.java:436)
	at java.security.AccessController.doPrivileged(java.base@11.0.6/Native Method)
	at sun.nio.ch.IOUtil.<clinit>(java.base@11.0.6/IOUtil.java:435)
	at sun.nio.ch.FileChannelImpl.<clinit>(java.base@11.0.6/FileChannelImpl.java:1212)
	at sun.nio.fs.UnixChannelFactory.newFileChannel(java.base@11.0.6/UnixChannelFactory.java:144)
	at sun.nio.fs.UnixChannelFactory.newFileChannel(java.base@11.0.6/UnixChannelFactory.java:156)
	at sun.nio.fs.UnixFileSystemProvider.newByteChannel(java.base@11.0.6/UnixFileSystemProvider.java:217)
	at java.nio.file.Files.newByteChannel(java.base@11.0.6/Files.java:370)
	at java.nio.file.Files.newByteChannel(java.base@11.0.6/Files.java:421)
	at java.nio.file.spi.FileSystemProvider.newInputStream(java.base@11.0.6/FileSystemProvider.java:420)
	at java.nio.file.Files.newInputStream(java.base@11.0.6/Files.java:155)
	at javax.crypto.JceSecurity.setupJurisdictionPolicies(java.base@11.0.6/JceSecurity.java:335)
	at javax.crypto.JceSecurity$1.run(java.base@11.0.6/JceSecurity.java:111)
	at javax.crypto.JceSecurity$1.run(java.base@11.0.6/JceSecurity.java:108)
	at java.security.AccessController.doPrivileged(java.base@11.0.6/Native Method)
	at javax.crypto.JceSecurity.<clinit>(java.base@11.0.6/JceSecurity.java:107)
	at javax.crypto.Cipher.getInstance(java.base@11.0.6/Cipher.java:540)
	at sun.security.ssl.JsseJce.getCipher(java.base@11.0.6/JsseJce.java:185)
	at sun.security.ssl.SSLCipher.isTransformationAvailable(java.base@11.0.6/SSLCipher.java:483)
	at sun.security.ssl.SSLCipher.<init>(java.base@11.0.6/SSLCipher.java:472)
	at sun.security.ssl.SSLCipher.<clinit>(java.base@11.0.6/SSLCipher.java:81)
	at sun.security.ssl.CipherSuite.<clinit>(java.base@11.0.6/CipherSuite.java:67)
	at sun.security.ssl.SSLContextImpl.getApplicableSupportedCipherSuites(java.base@11.0.6/SSLContextImpl.java:348)
	at sun.security.ssl.SSLContextImpl$AbstractTLSContext.<clinit>(java.base@11.0.6/SSLContextImpl.java:580)
	at java.lang.Class.forName0(java.base@11.0.6/Native Method)
	at java.lang.Class.forName(java.base@11.0.6/Class.java:315)
	at java.security.Provider$Service.getImplClass(java.base@11.0.6/Provider.java:1848)
	at java.security.Provider$Service.newInstance(java.base@11.0.6/Provider.java:1824)
	at sun.security.jca.GetInstance.getInstance(java.base@11.0.6/GetInstance.java:236)
	at sun.security.jca.GetInstance.getInstance(java.base@11.0.6/GetInstance.java:164)
	at javax.net.ssl.SSLContext.getInstance(java.base@11.0.6/SSLContext.java:168)
	at javax.net.ssl.SSLContext.getDefault(java.base@11.0.6/SSLContext.java:99)
	- locked <0x00000007bb3f0528> (a java.lang.Class for javax.net.ssl.SSLContext)
	at javax.net.ssl.SSLSocketFactory.getDefault(java.base@11.0.6/SSLSocketFactory.java:123)
	- locked <0x00000007bb3f02e0> (a java.lang.Class for javax.net.ssl.SSLSocketFactory)
	at javax.net.ssl.HttpsURLConnection.getDefaultSSLSocketFactory(java.base@11.0.6/HttpsURLConnection.java:335)
	at javax.net.ssl.HttpsURLConnection.<init>(java.base@11.0.6/HttpsURLConnection.java:292)
	at sun.net.www.protocol.https.HttpsURLConnectionImpl.<init>(java.base@11.0.6/HttpsURLConnectionImpl.java:100)
	at sun.net.www.protocol.https.Handler.openConnection(java.base@11.0.6/Handler.java:62)
	at sun.net.www.protocol.https.Handler.openConnection(java.base@11.0.6/Handler.java:57)
	at java.net.URL.openConnection(java.base@11.0.6/URL.java:1074)

"Thread-2" #30 prio=5 os_prio=0 cpu=65.77ms elapsed=259.92s tid=0x000014d2d1450800 nid=0x32e672 in Object.wait()  [0x000014d24d1a5000]
   java.lang.Thread.State: RUNNABLE
	at java.lang.ClassLoader$NativeLibrary.load0(java.base@11.0.6/Native Method)
	at java.lang.ClassLoader$NativeLibrary.load(java.base@11.0.6/ClassLoader.java:2430)
	at java.lang.ClassLoader$NativeLibrary.loadLibrary(java.base@11.0.6/ClassLoader.java:2487)
	- locked <0x00000007b8cf1178> (a java.util.HashSet)
	at java.lang.ClassLoader.loadLibrary0(java.base@11.0.6/ClassLoader.java:2684)
	at java.lang.ClassLoader.loadLibrary(java.base@11.0.6/ClassLoader.java:2649)
	at java.lang.Runtime.loadLibrary0(java.base@11.0.6/Runtime.java:829)
	- locked <0x00000007b88cb060> (a java.lang.Runtime)
	at java.lang.System.loadLibrary(java.base@11.0.6/System.java:1870)
	at io.netty.util.internal.NativeLibraryUtil.loadLibrary(NativeLibraryUtil.java:38)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@11.0.6/Native Method)
	at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@11.0.6/NativeMethodAccessorImpl.java:62)
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@11.0.6/DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(java.base@11.0.6/Method.java:566)
	at io.netty.util.internal.NativeLibraryLoader$1.run(NativeLibraryLoader.java:369)
	at java.security.AccessController.doPrivileged(java.base@11.0.6/Native Method)
	at io.netty.util.internal.NativeLibraryLoader.loadLibraryByHelper(NativeLibraryLoader.java:361)
	at io.netty.util.internal.NativeLibraryLoader.loadLibrary(NativeLibraryLoader.java:339)
	at io.netty.util.internal.NativeLibraryLoader.load(NativeLibraryLoader.java:136)
	at io.netty.channel.epoll.Native.loadNativeLibrary(Native.java:186)
	at io.netty.channel.epoll.Native.<clinit>(Native.java:57)
	at io.netty.channel.epoll.Epoll.<clinit>(Epoll.java:39)
	at io.netty.channel.epoll.EpollEventLoop.<clinit>(EpollEventLoop.java:53)
	at io.netty.channel.epoll.EpollEventLoopGroup.newChild(EpollEventLoopGroup.java:143)
	at io.netty.channel.epoll.EpollEventLoopGroup.newChild(EpollEventLoopGroup.java:36)
	at io.netty.util.concurrent.MultithreadEventExecutorGroup.<init>(MultithreadEventExecutorGroup.java:84)
	at io.netty.util.concurrent.MultithreadEventExecutorGroup.<init>(MultithreadEventExecutorGroup.java:58)
	at io.netty.util.concurrent.MultithreadEventExecutorGroup.<init>(MultithreadEventExecutorGroup.java:47)
	at io.netty.channel.MultithreadEventLoopGroup.<init>(MultithreadEventLoopGroup.java:59)
	at io.netty.channel.epoll.EpollEventLoopGroup.<init>(EpollEventLoopGroup.java:105)
	at io.netty.channel.epoll.EpollEventLoopGroup.<init>(EpollEventLoopGroup.java:92)
	at io.netty.channel.epoll.EpollEventLoopGroup.<init>(EpollEventLoopGroup.java:69)
	at io.netty.channel.epoll.EpollEventLoopGroup.<init>(EpollEventLoopGroup.java:53)
	at io.netty.channel.epoll.EpollEventLoopGroup.<init>(EpollEventLoopGroup.java:46)

Netty version

4.1.39.Final

JVM version (e.g. java -version)

OpenJDK 11.0.6

OS version (e.g. uname -a)

Linux 4.14.67 on x86_64