spring-cloud-dataflow: SCDF server fails to access docker hub behind HTTP proxy

Description: With version 2.5.3 setting https.proxyHost, https.proxyPort and http.nonProxyHosts system properties was enough to let the server to talk outside. For example, it could retrieve metadata of docker images in Docker Hub.

We are testing 2.7.0 now and this is not enough anymore. ApplicationConfigurationMetadataResolver catches Connection timed out exception on opening of /dashboard/#/apps/source/<app name>. This would not be a big problem, but with 2.5.3 the misconfigured proxy was causing failures to deploy Streams, which using applications with docker images from Docker hub.

Release versions:

Installed in Kubernetes using Bitnami Helm chart

$ curl 'http://localhost:8080/about' -i -X GET -H 'Accept: application/json'
HTTP/1.1 200
Content-Type: application/json
Transfer-Encoding: chunked
Date: Mon, 21 Dec 2020 15:57:34 GMT

{"featureInfo":{"analyticsEnabled":true,"streamsEnabled":true,"tasksEnabled":true,"schedulesEnabled":true,"monitoringDashboardType":"NONE"},"versionInfo":{"implementation":{"name":"spring-cloud-dataflow-server","version":"2.7.0"},"core":{"name":"Spring Cloud Data Flow Core","version":"2.7.0"},"dashboard":{"name":"Spring Cloud Dataflow UI","version":"3.0.0"},"shell":{"name":"Spring Cloud Data Flow Shell","version":"2.7.0","url":"https://repo1.maven.org/maven2/org/springframework/cloud/spring-cloud-dataflow-shell/2.7.0/spring-cloud-dataflow-shell-2.7.0.jar"}},"securityInfo":{"authenticationEnabled":false,"authenticated":false,"username":null,"roles":[]},"runtimeEnvironment":{"appDeployer":{"deployerImplementationVersion":"2.6.0","deployerName":"Spring Cloud Skipper Server","deployerSpiVersion":"2.6.0","javaVersion":"1.8.0_275","platformApiVersion":"","platformClientVersion":"","platformHostVersion":"","platformSpecificInfo":{"default":"kubernetes","data-pipelines":"kubernetes","kime":"kubernetes","data-enrichment":"kubernetes","patstat-bf":"kubernetes","search-process":"kubernetes","search-service-layer":"kubernetes","inbox":"kubernetes","aly":"kubernetes"},"platformType":"Skipper Managed","springBootVersion":"2.3.5.RELEASE","springVersion":"5.2.10.RELEASE"},"taskLaunchers":[{"deployerImplementationVersion":"2.5.0","deployerName":"KubernetesTaskLauncher","deployerSpiVersion":"2.5.0","javaVersion":"1.8.0_275","platformApiVersion":"v1","platformClientVersion":"unknown","platformHostVersion":"unknown","platformSpecificInfo":{"namespace":"search-process","master-url":"https://172.17.224.1:443/"},"platformType":"Kubernetes","springBootVersion":"2.3.5.RELEASE","springVersion":"5.2.10.RELEASE"},{"deployerImplementationVersion":"2.5.0","deployerName":"KubernetesTaskLauncher","deployerSpiVersion":"2.5.0","javaVersion":"1.8.0_275","platformApiVersion":"v1","platformClientVersion":"unknown","platformHostVersion":"unknown","platformSpecificInfo":{"namespace":"data-enrichment","master-url":"https://172.17.224.1:443/"},"platformType":"Kubernetes","springBootVersion":"2.3.5.RELEASE","springVersion":"5.2.10.RELEASE"},{"deployerImplementationVersion":"2.5.0","deployerName":"KubernetesTaskLauncher","deployerSpiVersion":"2.5.0","javaVersion":"1.8.0_275","platformApiVersion":"v1","platformClientVersion":"unknown","platformHostVersion":"unknown","platformSpecificInfo":{"namespace":"kime","master-url":"https://172.17.224.1:443/"},"platformType":"Kubernetes","springBootVersion":"2.3.5.RELEASE","springVersion":"5.2.10.RELEASE"},{"deployerImplementationVersion":"2.5.0","deployerName":"KubernetesTaskLauncher","deployerSpiVersion":"2.5.0","javaVersion":"1.8.0_275","platformApiVersion":"v1","platformClientVersion":"unknown","platformHostVersion":"unknown","platformSpecificInfo":{"namespace":"kime","master-url":"https://172.17.224.1:443/"},"platformType":"Kubernetes","springBootVersion":"2.3.5.RELEASE","springVersion":"5.2.10.RELEASE"},{"deployerImplementationVersion":"2.5.0","deployerName":"KubernetesTaskLauncher","deployerSpiVersion":"2.5.0","javaVersion":"1.8.0_275","platformApiVersion":"v1","platformClientVersion":"unknown","platformHostVersion":"unknown","platformSpecificInfo":{"namespace":"inbox","master-url":"https://172.17.224.1:443/"},"platformType":"Kubernetes","springBootVersion":"2.3.5.RELEASE","springVersion":"5.2.10.RELEASE"},{"deployerImplementationVersion":"2.5.0","deployerName":"KubernetesTaskLauncher","deployerSpiVersion":"2.5.0","javaVersion":"1.8.0_275","platformApiVersion":"v1","platformClientVersion":"unknown","platformHostVersion":"unknown","platformSpecificInfo":{"namespace":"aly","master-url":"https://172.17.224.1:443/"},"platformType":"Kubernetes","springBootVersion":"2.3.5.RELEASE","springVersion":"5.2.10.RELEASE"},{"deployerImplementationVersion":"2.5.0","deployerName":"KubernetesTaskLauncher","deployerSpiVersion":"2.5.0","javaVersion":"1.8.0_275","platformApiVersion":"v1","platformClientVersion":"unknown","platformHostVersion":"unknown","platformSpecificInfo":{"namespace":"data-pipelines","master-url":"https://172.17.224.1:443/"},"platformType":"Kubernetes","springBootVersion":"2.3.5.RELEASE","springVersion":"5.2.10.RELEASE"},{"deployerImplementationVersion":"2.5.0","deployerName":"KubernetesTaskLauncher","deployerSpiVersion":"2.5.0","javaVersion":"1.8.0_275","platformApiVersion":"v1","platformClientVersion":"unknown","platformHostVersion":"unknown","platformSpecificInfo":{"namespace":"patstat-bf","master-url":"https://172.17.224.1:443/"},"platformType":"Kubernetes","springBootVersion":"2.3.5.RELEASE","springVersion":"5.2.10.RELEASE"},{"deployerImplementationVersion":"2.5.0","deployerName":"KubernetesTaskLauncher","deployerSpiVersion":"2.5.0","javaVersion":"1.8.0_275","platformApiVersion":"v1","platformClientVersion":"unknown","platformHostVersion":"unknown","platformSpecificInfo":{"namespace":"search-service-layer","master-url":"https://172.17.224.1:443/"},"platformType":"Kubernetes","springBootVersion":"2.3.5.RELEASE","springVersion":"5.2.10.RELEASE"}]},"monitoringDashboardInfo":{"url":"","refreshInterval":15,"dashboardType":"NONE","source":"default-scdf-source"},"_links":{"self":{"href":"http://localhost:8080/about"}}}

Additional context:

The relatively new ContainerRegistryAutoConfiguration uses HttpClients.custom(), not HttpClients.createSystem(). Could this be the reason? https://github.com/spring-cloud/spring-cloud-dataflow/blob/8644a5cdef59e5fba6456c200ef626575820862b/spring-cloud-dataflow-container-registry/src/main/java/org/springframework/cloud/dataflow/container/registry/ContainerRegistryAutoConfiguration.java#L187

Exception trace:

2020-12-21 15:03:33.713  WARN 1 --- [nio-8080-exec-3] ApplicationConfigurationMetadataResolver : Failed to retrieve properties for resource Docker Resource [docker:springcloudstream/file-source-kafka:2.1.2.RELEASE] because of ConnectException: Connection timed out (Connection timed out)
2020-12-21 15:03:33.718 DEBUG 1 --- [nio-8080-exec-3] ApplicationConfigurationMetadataResolver : (Details) for failed to retrieve properties for resource:Docker Resource [docker:springcloudstream/file-source-kafka:2.1.2.RELEASE]

org.springframework.web.client.ResourceAccessException: I/O error on GET request for "https://auth.docker.io/token": Connect to auth.docker.io:443 [auth.docker.io/52.5.11.128, auth.docker.io/52.1.121.53, auth.docker.io/107.23.149.57, auth.docker.io/18.232.227.119, auth.docker.io/54.236.131.166, auth.docker.io/3.218.162.19, auth.docker.io/52.72.232.213, auth.docker.io/52.20.56.50] failed: Connection timed out (Connection timed out); nested exception is org.apache.http.conn.HttpHostConnectException: Connect to auth.docker.io:443 [auth.docker.io/52.5.11.128, auth.docker.io/52.1.121.53, auth.docker.io/107.23.149.57, auth.docker.io/18.232.227.119, auth.docker.io/54.236.131.166, auth.docker.io/3.218.162.19, auth.docker.io/52.72.232.213, auth.docker.io/52.20.56.50] failed: Connection timed out (Connection timed out)
	at org.springframework.web.client.RestTemplate.doExecute(RestTemplate.java:746)
	at org.springframework.web.client.RestTemplate.execute(RestTemplate.java:712)
	at org.springframework.web.client.RestTemplate.exchange(RestTemplate.java:600)
	at org.springframework.cloud.dataflow.container.registry.authorization.DockerOAuth2RegistryAuthorizer.getAuthorizationHeaders(DockerOAuth2RegistryAuthorizer.java:116)
    .......................................................................................
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.http.conn.HttpHostConnectException: Connect to auth.docker.io:443 [auth.docker.io/52.5.11.128, auth.docker.io/52.1.121.53, auth.docker.io/107.23.149.57, auth.docker.io/18.232.227.119, auth.docker.io/54.236.131.166, auth.docker.io/3.218.162.19, auth.docker.io/52.72.232.213, auth.docker.io/52.20.56.50] failed: Connection timed out (Connection timed out)
	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:156)
	at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:376)
	at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:393)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.springframework.http.client.HttpComponentsClientHttpRequest.executeInternal(HttpComponentsClientHttpRequest.java:87)
	at org.springframework.http.client.AbstractBufferingClientHttpRequest.executeInternal(AbstractBufferingClientHttpRequest.java:48)
	at org.springframework.http.client.AbstractClientHttpRequest.execute(AbstractClientHttpRequest.java:53)
	at org.springframework.http.client.InterceptingClientHttpRequest$InterceptingRequestExecution.execute(InterceptingClientHttpRequest.java:109)
	at org.springframework.boot.actuate.metrics.web.client.MetricsClientHttpRequestInterceptor.intercept(MetricsClientHttpRequestInterceptor.java:81)
	at org.springframework.http.client.InterceptingClientHttpRequest$InterceptingRequestExecution.execute(InterceptingClientHttpRequest.java:93)
	at org.springframework.http.client.InterceptingClientHttpRequest.executeInternal(InterceptingClientHttpRequest.java:77)
	at org.springframework.http.client.AbstractBufferingClientHttpRequest.executeInternal(AbstractBufferingClientHttpRequest.java:48)
	at org.springframework.http.client.AbstractClientHttpRequest.execute(AbstractClientHttpRequest.java:53)
	at org.springframework.web.client.RestTemplate.doExecute(RestTemplate.java:737)
	... 68 common frames omitted
Caused by: java.net.ConnectException: Connection timed out (Connection timed out)
	at java.net.PlainSocketImpl.socketConnect(Native Method)
	at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
	at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
	at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
	at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
	at java.net.Socket.connect(Socket.java:607)
	at org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:368)
	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:142)
	... 87 common frames omitted