spring-boot: @Async annotation breaks Trace continuity

Spring Boot: 3.0.3

Using method with @Async annotation causes that Trace context is broken and spans within Async method are not child for outer spans.

Regarding this guide: https://github.com/micrometer-metrics/tracing/wiki/Spring-Cloud-Sleuth-3.1-Migration-Guide#async-instrumentation it should be working when declaring specific ThreadPoolTaskScheduler. But even with it, it is not working as expected. Sample test:

package com.example;

import io.micrometer.context.ContextExecutorService;
import io.micrometer.context.ContextSnapshot;
import io.micrometer.tracing.Span;
import io.micrometer.tracing.Tracer;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.test.autoconfigure.actuate.observability.AutoConfigureObservability;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.task.SimpleAsyncTaskExecutor;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.AsyncConfigurer;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
import org.springframework.stereotype.Component;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.servlet.config.annotation.AsyncSupportConfigurer;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;

import java.util.concurrent.*;

import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.RANDOM_PORT;

@AutoConfigureObservability
@SpringBootTest(classes = {AsyncTracingTest.AsyncTracingTestApplication.class}, webEnvironment = RANDOM_PORT, properties = {
        "spring.cloud.gcp.trace.enabled=true",
        "spring.cloud.gcp.core.enabled=true",
        "spring.cloud.gcp.trace.project-id=xyz-project-id"
})
class AsyncTracingTest {
    @Autowired
    Tracer tracer;

    @Autowired
    AsyncTracingTestApplication.AsyncLogic asyncLogic;

    @Test
    void asyncTracingTest() {
        Span firstSpan = tracer.nextSpan().name("First span").tag("test", "test");
        try (Tracer.SpanInScope scope = this.tracer.withSpan(firstSpan.start())) {
            asyncLogic.asyncCall();
        }
        finally {
            firstSpan.end();
        }
    }

    @EnableAsync
    @SpringBootApplication(scanBasePackages = {"scanThisClassOnly"})
    @Configuration(proxyBeanMethods = false)
    static class AsyncTracingTestApplication {

        @Configuration(proxyBeanMethods = false)
        static class AsyncConfig implements AsyncConfigurer, WebMvcConfigurer {
            @Override
            public Executor getAsyncExecutor() {
                return ContextExecutorService.wrap(Executors.newCachedThreadPool(), ContextSnapshot::captureAll);
            }

            @Override
            public void configureAsyncSupport(AsyncSupportConfigurer configurer) {
                configurer.setTaskExecutor(new SimpleAsyncTaskExecutor(r -> new Thread(ContextSnapshot.captureAll().wrap(r))));
            }
        }

        @Bean(name = "taskExecutor", destroyMethod = "shutdown")
        ThreadPoolTaskScheduler threadPoolTaskScheduler() {
            ThreadPoolTaskScheduler threadPoolTaskScheduler = new ThreadPoolTaskScheduler() {
                @Override
                protected ExecutorService initializeExecutor(ThreadFactory threadFactory, RejectedExecutionHandler rejectedExecutionHandler) {
                    ExecutorService executorService = super.initializeExecutor(threadFactory, rejectedExecutionHandler);
                    return ContextExecutorService.wrap(executorService, ContextSnapshot::captureAll);
                }
            };
            threadPoolTaskScheduler.initialize();
            return threadPoolTaskScheduler;
        }

        @Component
        class AsyncLogic {
            Tracer tracer;

            public AsyncLogic(Tracer tracer) {
                this.tracer = tracer;
            }

            @Async("taskExecutor")
            public void asyncCall() {
                tracer.nextSpan().name("Second span").tag("test", "test").start().end();
            }
        }
    }
}

It results in 2 separate traces for First and Second spans: image

After removing @Async annotation it is working as expected for synchronous call: image

On Spring Boot 2.X with Sleuth it was working as continue trace (with additional span for async call). Sample test:

package com.example;

import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.cloud.sleuth.Span;
import org.springframework.cloud.sleuth.Tracer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.task.SimpleAsyncTaskExecutor;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.AsyncConfigurer;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
import org.springframework.stereotype.Component;
import org.springframework.web.servlet.config.annotation.AsyncSupportConfigurer;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;

import java.util.concurrent.*;

import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.RANDOM_PORT;

@SpringBootTest(classes = {AsyncTracingTest.AsyncTracingTestApplication.class}, webEnvironment = RANDOM_PORT, properties = {
        "spring.cloud.gcp.trace.enabled=true",
        "spring.cloud.gcp.core.enabled=true",
        "spring.cloud.gcp.trace.project-id=xyz-project-id"
})
class AsyncTracingTest {
    @Autowired
    Tracer tracer;

    @Autowired
    AsyncTracingTestApplication.AsyncLogic asyncLogic;

    @Test
    void asyncTracingTest() {
        Span firstSpan = tracer.nextSpan().name("First span").tag("test", "test");
        try (Tracer.SpanInScope scope = this.tracer.withSpan(firstSpan.start())) {
            asyncLogic.asyncCall();
        }
        finally {
            firstSpan.end();
        }
    }

    @EnableAsync
    @SpringBootApplication(scanBasePackages = {"scanThisClassOnly"})
    @Configuration(proxyBeanMethods = false)
    static class AsyncTracingTestApplication {

        @Component
        class AsyncLogic {
            Tracer tracer;

            public AsyncLogic(Tracer tracer) {
                this.tracer = tracer;
            }
            @Async("taskExecutor")
            public void asyncCall() {
                tracer.nextSpan().name("Second span").tag("test", "test").start().end();
            }
        }
    }
}

image

For me it seems like a bug with @Async handling. E.g. for AsyncTaskExecutor

Callable<String> tracedCallable = tracer.currentTraceContext().wrap(callable);
threadPoolTaskExecutor.submit(tracedCallable);

it correctly passes Tracing context to asynchronous callable.

About this issue

  • Original URL
  • State: closed
  • Created 4 years ago
  • Comments: 16 (7 by maintainers)

Most upvoted comments

@bylidev what gives you the impression that we don’t like this use case?

The workaround you have shared is not advised as it wraps the original type as ExecutorService and erases the concrete type ThreadPoolTaskExecutor. Exposing the most concrete type in the bean method signature is the preferred approach in Spring applications.

Setting up context propagation by default for all executors would create significant overhead, even if the use case doesn’t require it. You can check out Spring Framework’s reference documentation on observability for application events and executors for a simpler solution.

There were various problems after upgrading from Spring Cloud Sleuth to Micrometer Tracing. Using Spring Cloud Sleuth only requires a simple configuration, and switching to Micrometer Tracing makes me very upset.

Thank you @rafal-dudek for filing this issue and for a great reproducer.

Missing Wiki Entry

To begin with we’re missing wrapping of the ScheduledExecutorService in the wiki migration guide snippet (I’ve already updated the wiki page).

/**
     * NAME OF THE BEAN IS IMPORTANT!
     * <p>
     * We need to wrap this for @Async related things to propagate the context.
     *
     * @see EnableAsync
     */
    // [Observability] instrumenting executors
    @Bean(name = "taskExecutor", destroyMethod = "shutdown")
    ThreadPoolTaskScheduler threadPoolTaskScheduler() {
        ThreadPoolTaskScheduler threadPoolTaskScheduler = new ThreadPoolTaskScheduler() {
            @Override
            protected ExecutorService initializeExecutor(ThreadFactory threadFactory, RejectedExecutionHandler rejectedExecutionHandler) {
                ExecutorService executorService = super.initializeExecutor(threadFactory, rejectedExecutionHandler);
                return ContextExecutorService.wrap(executorService, ContextSnapshot::captureAll);
            }

                // THIS WAS MISSING
                @Override
                public ScheduledExecutorService getScheduledExecutor() throws IllegalStateException {
                    return ContextScheduledExecutorService.wrap(super.getScheduledExecutor());
                }
        };
        threadPoolTaskScheduler.initialize();
        return threadPoolTaskScheduler;
    }

Making This Work With Current Setup

Adding that snippet won’t help you out fully just yet. In order for this to work out of the box, you would need to change your code to use observations as presented below:

@Test
    void asyncTracingTest() {
        Observation firstSpan = Observation.createNotStarted("First span", observationRegistry).highCardinalityKeyValue("test", "test");
        try (Observation.Scope scope = firstSpan.start().openScope()) {
            log.info("Async in test with observation - before call");          
            asyncLogic.asyncCall();
            log.info("Async in test with observation - after call");
        }
        finally {
            firstSpan.stop();
        }

        Future<Boolean> submit = threadPoolTaskScheduler.submit(() -> {
			log.info("There should be no span here");
			return tracer.currentSpan() == null;
		});
		boolean noCurrentSpan = submit.get(1, TimeUnit.SECONDS);

		Assertions.assertThat(noCurrentSpan).isTrue();
    }

If you do that you will observe that the ids got propagated.

Making This Work With Micrometer Tracing API

However there should be no problem with actually using just the Micrometer Tracing API and assuming that things just work. So let me suggest a temporary workaround for you until we figure out a better way of doing things.

What we will need is a Micrometer Tracing ThreadLocalAccessor that will be aware of Observations. You can use this code:

/*
 * Copyright 2002-2021 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.example.boot34622;

import io.micrometer.context.ThreadLocalAccessor;
import io.micrometer.observation.Observation;
import io.micrometer.observation.ObservationRegistry;
import io.micrometer.observation.contextpropagation.ObservationThreadLocalAccessor;
import io.micrometer.tracing.Span;
import io.micrometer.tracing.Tracer;
import io.micrometer.tracing.handler.TracingObservationHandler;

/**
 * A {@link ThreadLocalAccessor} to put and restore current {@link Span} depending on whether {@link ObservationThreadLocalAccessor} did some
 * work or not.
 *
 * @author Marcin Grzejszczak
 */
public class ObservationAwareSpanThreadLocalAccessor implements ThreadLocalAccessor<Span> {

    /**
     * Key under which this accessor is being registered.
     */
    public static final String KEY = "micrometer.tracing";

    private final Tracer tracer;

    private static final ObservationRegistry registry = ObservationRegistry.create();

    public ObservationAwareSpanThreadLocalAccessor(Tracer tracer) {
        this.tracer = tracer;
    }

    @Override
    public Object key() {
        return KEY;
    }

    @Override
    public Span getValue() {
        Observation currentObservation = registry.getCurrentObservation();
        if (currentObservation != null) {
            // There's a current observation so OTLA hooked in
            // we will now check if the user created spans manually or not
            TracingObservationHandler.TracingContext tracingContext = currentObservation.getContextView().getOrDefault(TracingObservationHandler.TracingContext.class, new TracingObservationHandler.TracingContext());
            Span currentSpan = tracer.currentSpan();
            if (currentSpan != null && !currentSpan.equals(tracingContext.getSpan())) {
                // User created child spans manually and scoped them
                // the current span is not the same as the one from observation
                return currentSpan;
            }
            // Current span is same as the one from observation, we will skip this
            return null;
        }
        // No current observation so let's check the tracer
        return this.tracer.currentSpan();
    }

    @Override
    public void setValue(Span value) {
        this.tracer.withSpan(value);
    }

    @Override
    public void reset() {
        this.tracer.withSpan(null);
    }
}

Having that you need to register it in your code. You can do it by calling the Context Propagation API like this


@Configuration(proxyBeanMethods = false)
class Config {

        @Autowired
        Tracer tracer;

        @PostConstruct
        void setup() {
             ContextRegistry.getInstance().registerThreadLocalAccessor(new ObservationAwareSpanThreadLocalAccessor(tracer));
        }
}

Test Code

With that done tracing context will be propagated regardless of the fact whether you’re using Observation API and / or Tracing API. Let me paste the whole testing code:

package com.example.boot34622;

import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.RejectedExecutionHandler;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;

import io.micrometer.context.ContextExecutorService;
import io.micrometer.context.ContextRegistry;
import io.micrometer.context.ContextScheduledExecutorService;
import io.micrometer.context.ContextSnapshot;
import io.micrometer.observation.Observation;
import io.micrometer.observation.ObservationRegistry;
import io.micrometer.tracing.Span;
import io.micrometer.tracing.Tracer;
import jakarta.annotation.PostConstruct;
import org.assertj.core.api.Assertions;
import org.assertj.core.api.BDDAssertions;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.test.autoconfigure.actuate.observability.AutoConfigureObservability;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.task.SimpleAsyncTaskExecutor;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.AsyncConfigurer;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
import org.springframework.stereotype.Component;
import org.springframework.web.servlet.config.annotation.AsyncSupportConfigurer;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;

import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.RANDOM_PORT;

@AutoConfigureObservability
@SpringBootTest(classes = {AsyncTracingTest.AsyncTracingTestApplication.class}, webEnvironment = RANDOM_PORT, properties = {
		"spring.cloud.gcp.trace.enabled=true",
		"spring.cloud.gcp.core.enabled=true",
		"spring.cloud.gcp.trace.project-id=xyz-project-id"
})
class AsyncTracingTest {

	static final Logger log = LoggerFactory.getLogger(AsyncTracingTest.class);

	@Autowired
	Tracer tracer;

	@Autowired
	ObservationRegistry observationRegistry;

	@Autowired
	AsyncTracingTestApplication.AsyncLogic asyncLogic;

	@Autowired
	ThreadPoolTaskScheduler threadPoolTaskScheduler;

	@Test
	void asyncTracingTestWithObservationAndManualSpans() throws ExecutionException, InterruptedException, TimeoutException {
		Observation firstSpan = Observation.createNotStarted("First span", observationRegistry).highCardinalityKeyValue("test", "test");
		try (Observation.Scope scope = firstSpan.start().openScope()) {
			log.info("Async in test with observation - before call");

			Span secondSpan = tracer.nextSpan().name("Second span").tag("test", "test");
			try (Tracer.SpanInScope scope2 = this.tracer.withSpan(secondSpan.start())) {
				log.info("Async in test with span - before call");
				Future<String> future = asyncLogic.asyncCall();
				String spanIdFromFuture = future.get(1, TimeUnit.SECONDS);
				log.info("Async in test with span - after call");
				BDDAssertions.then(spanIdFromFuture).isEqualTo(secondSpan.context().spanId());
			}
			finally {
				secondSpan.end();
			}

			log.info("Async in test with observation - after call");
		}
		finally {
			firstSpan.stop();
		}

		Future<Boolean> submit = threadPoolTaskScheduler.submit(() -> {
			log.info("There should be no span here");
			return tracer.currentSpan() == null;
		});
		boolean noCurrentSpan = submit.get(1, TimeUnit.SECONDS);

		Assertions.assertThat(noCurrentSpan).isTrue();

	}

	@Test
	void asyncTracingTestWithJustObservation() throws ExecutionException, InterruptedException, TimeoutException {
		Observation firstObservation = Observation.createNotStarted("First span", observationRegistry).highCardinalityKeyValue("test", "test");
		try (Observation.Scope scope = firstObservation.start().openScope()) {
			log.info("Async in test with span - before call");
			String currentSpanId = tracer.currentSpan().context().spanId();
			Future<String> future = asyncLogic.asyncCall();
			String spanIdFromFuture = future.get(1, TimeUnit.SECONDS);
			log.info("Async in test with span - after call");
			BDDAssertions.then(spanIdFromFuture).isEqualTo(currentSpanId);
		}
		finally {
			firstObservation.stop();
		}

		Future<Boolean> submit = threadPoolTaskScheduler.submit(() -> {
			log.info("There should be no span here");
			return tracer.currentSpan() == null;
		});
		boolean noCurrentSpan = submit.get(1, TimeUnit.SECONDS);

		Assertions.assertThat(noCurrentSpan).isTrue();

	}

	@Test
	void asyncTracingTestWithJustSpans() throws ExecutionException, InterruptedException, TimeoutException {
		Span secondSpan = tracer.nextSpan().name("Second span").tag("test", "test");
		try (Tracer.SpanInScope scope2 = this.tracer.withSpan(secondSpan.start())) {
			log.info("Async in test with span - before call");
			Future<String> future = asyncLogic.asyncCall();
			String spanIdFromFuture = future.get(1, TimeUnit.SECONDS);
			log.info("Async in test with span - after call");
			BDDAssertions.then(spanIdFromFuture).isEqualTo(secondSpan.context().spanId());
		}
		finally {
			secondSpan.end();
		}

		Future<Boolean> submit = threadPoolTaskScheduler.submit(() -> {
			log.info("There should be no span here");
			return tracer.currentSpan() == null;
		});
		boolean noCurrentSpan = submit.get(1, TimeUnit.SECONDS);

		Assertions.assertThat(noCurrentSpan).isTrue();

	}

	@EnableAsync
	@SpringBootApplication(scanBasePackages = {"scanThisClassOnly"})
	@Configuration(proxyBeanMethods = false)
	static class AsyncTracingTestApplication {

		@Configuration(proxyBeanMethods = false)
		static class AsyncConfig implements AsyncConfigurer, WebMvcConfigurer {
			@Override
			public Executor getAsyncExecutor() {
				return ContextExecutorService.wrap(Executors.newCachedThreadPool(), ContextSnapshot::captureAll);
			}

			@Override
			public void configureAsyncSupport(AsyncSupportConfigurer configurer) {
				configurer.setTaskExecutor(new SimpleAsyncTaskExecutor(r -> new Thread(ContextSnapshot.captureAll().wrap(r))));
			}
		}

		@Bean(name = "taskExecutor", destroyMethod = "shutdown")
		ThreadPoolTaskScheduler threadPoolTaskScheduler() {
			ThreadPoolTaskScheduler threadPoolTaskScheduler = new ThreadPoolTaskScheduler() {
				@Override
				protected ExecutorService initializeExecutor(ThreadFactory threadFactory, RejectedExecutionHandler rejectedExecutionHandler) {
					ExecutorService executorService = super.initializeExecutor(threadFactory, rejectedExecutionHandler);
					return ContextExecutorService.wrap(executorService, ContextSnapshot::captureAll);
				}

				@Override
				public ScheduledExecutorService getScheduledExecutor() throws IllegalStateException {
					return ContextScheduledExecutorService.wrap(super.getScheduledExecutor());
				}
			};
			threadPoolTaskScheduler.initialize();
			return threadPoolTaskScheduler;
		}

		@Autowired
		Tracer tracer;

		@PostConstruct
		void setup() {
			ContextRegistry.getInstance().registerThreadLocalAccessor(new ObservationAwareSpanThreadLocalAccessor(tracer));
		}

		@Component
		class AsyncLogic {
			Tracer tracer;

			public AsyncLogic(Tracer tracer) {
				this.tracer = tracer;
			}

			@Async("taskExecutor")
			public Future<String> asyncCall() {
				log.info("TASK EXECUTOR");
				tracer.nextSpan().name("Second span").tag("test", "test").start().end();
				String spanId = tracer.currentSpan().context().spanId();
				return CompletableFuture.supplyAsync(() -> spanId);
			}
		}
	}
}

Summary

WDYT about this? I’ve checked this code and it works fine, if that’s possible, could you please check it also on your side @rafal-dudek ?

I’ve created an issue in Micrometer Tracing to track this https://github.com/micrometer-metrics/tracing/issues/206.

Solved adding task decorator as well, thanks !

executor.setTaskDecorator(new ContextPropagatingTaskDecorator());

@marcingrzejszczak Will this work in Spring Boot 3.2.2 with virtual threads?

Yes it is working, didn’t thought to check that. So I am waiting for https://github.com/micrometer-metrics/tracing/issues/206. Thanks

Thanks, with ScheduledExecutorService override and using Observation instead of Span tracing for @Async is working properly.

I also checked your ObservationAwareSpanThreadLocalAccessor and it is working for Spans without Observation.

I’m looking forward to more automatic way to do that in some future Spring Boot update, as it might be convenient for our teams.