WSL: Windows commands inside WSL hang forever
Version
WSL 0.50.2.0
WSL Version
- WSL 2
- WSL 1
Kernel Version
5.10.74.3
Distro Version
Ubuntu 20.04.3 LTS
Other Software
No response
Repro Steps
I’m not sure what went wrong nor how to debug this. It was working fine before but all of a sudden I stopped being able to use windows commands from inside WSL. notepad.exe, wsl.exe, etc they all just hang forever and not even Ctrl+C is able to kill them and I’m forced to close the terminal.
Expected Behavior
Can use windows commands from inside WSL (e.g. wsl.exe --version)
Actual Behavior
Windows commands (e.g. wsl.exe --version) hang forever and not even Ctrl+C is able to kill them and I’m forced to close the terminal.
Diagnostic Logs
nodepad.exe
$ strace notepad.exe
execve("/mnt/c/Windows/system32/notepad.exe", ["notepad.exe"], 0x7fff357161e0 /* 37 vars */) = 0
arch_prctl(ARCH_SET_FS, 0x378290) = 0
set_tid_address(0x3782c8) = 400
brk(NULL) = 0xf79000
brk(0xf7a000) = 0xf7a000
brk(0xf7b000) = 0xf7b000
gettid() = 400
gettid() = 400
gettid() = 400
gettid() = 400
brk(0xf7c000) = 0xf7c000
brk(0xf7d000) = 0xf7d000
brk(0xf7e000) = 0xf7e000
sched_getaffinity(0, 128, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) = 32
getpid() = 400
getcwd("/home/tiago", 4096) = 12
uname({sysname="Linux", nodename="DESKTOP-VN589IP", ...}) = 0
getcwd("/home/tiago", 4096) = 12
open("/mnt/c/Windows/system32/notepad.exe", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_PATH) = 3
readlink("/proc/self/fd/3", "/mnt/c/Windows/system32/notepad."..., 4095) = 35
fstat(3, {st_mode=S_IFREG|0555, st_size=348160, ...}) = 0
stat("/mnt/c/Windows/system32/notepad.exe", {st_mode=S_IFREG|0555, st_size=348160, ...}) = 0
close(3) = 0
open("/proc/self/mountinfo", O_RDONLY) = 3
readv(3, [{iov_base="", iov_len=0}, {iov_base="68 72 0:26 / /mnt/wsl rw,relatim"..., iov_len=1024}], 2) = 1024
readv(3, [{iov_base="", iov_len=0}, {iov_base="- tmpfs none rw,mode=755\n96 95 0"..., iov_len=1024}], 2) = 1024
readv(3, [{iov_base="", iov_len=0}, {iov_base="ev,noexec,relatime - cgroup cgro"..., iov_len=1024}], 2) = 1024
readv(3, [{iov_base="", iov_len=0}, {iov_base="66 / /mnt/c rw,noatime - 9p drvf"..., iov_len=1024}], 2) = 270
readv(3, [{iov_base="", iov_len=0}, {iov_base="", iov_len=1024}], 2) = 0
close(3) = 0
getcwd("/home/tiago", 4096) = 12
open("/proc/self/mountinfo", O_RDONLY) = 3
readv(3, [{iov_base="", iov_len=0}, {iov_base="68 72 0:26 / /mnt/wsl rw,relatim"..., iov_len=1024}], 2) = 1024
readv(3, [{iov_base="", iov_len=0}, {iov_base="- tmpfs none rw,mode=755\n96 95 0"..., iov_len=1024}], 2) = 1024
readv(3, [{iov_base="", iov_len=0}, {iov_base="ev,noexec,relatime - cgroup cgro"..., iov_len=1024}], 2) = 1024
readv(3, [{iov_base="", iov_len=0}, {iov_base="66 / /mnt/c rw,noatime - 9p drvf"..., iov_len=1024}], 2) = 270
readv(3, [{iov_base="", iov_len=0}, {iov_base="", iov_len=1024}], 2) = 0
close(3) = 0
ioctl(0, TCGETS, {B38400 opost isig icanon echo ...}) = 0
ioctl(1, TCGETS, {B38400 opost isig icanon echo ...}) = 0
ioctl(2, TCGETS, {B38400 opost isig icanon echo ...}) = 0
ioctl(0, TIOCGPGRP, [397]) = 0
getpgid(0) = 397
fstat(0, {st_mode=S_IFCHR|0620, st_rdev=makedev(0x88, 0), ...}) = 0
fstat(1, {st_mode=S_IFCHR|0620, st_rdev=makedev(0x88, 0), ...}) = 0
fstat(2, {st_mode=S_IFCHR|0620, st_rdev=makedev(0x88, 0), ...}) = 0
ioctl(0, TIOCGWINSZ, {ws_row=39, ws_col=166, ws_xpixel=0, ws_ypixel=0}) = 0
ioctl(0, SNDCTL_TMR_START or TCSETS, {B38400 -opost -isig -icanon -echo ...}) = 0
dup(0) = 3
socket(AF_VSOCK, SOCK_STREAM|SOCK_CLOEXEC, 0) = 4
bind(4, {sa_family=AF_VSOCK, sa_data="\0\0\377\377\377\377\377\377\377\377\0\0\0\0"}, 16) = 0
getsockname(4, {sa_family=AF_VSOCK, sa_data="\0\0\343\221\264\332\377\377\377\377\0\0\0\0"}, [16]) = 0
listen(4, 4) = 0
socket(AF_UNIX, SOCK_SEQPACKET|SOCK_CLOEXEC, 0) = 5
connect(5, {sa_family=AF_UNIX, sun_path="/run/WSL/350_interop"}, 110) = 0
write(5, "\6\0\0\0006\1\0\0\343\221\264\332\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 310) = 310
poll([{fd=4, events=POLLIN}], 1, 10000) = 1 ([{fd=4, revents=POLLIN}])
accept4(4, {sa_family=AF_VSOCK, sa_data="\0\0]c\317O\2\0\0\0\0\0\0\0"}, [16], SOCK_CLOEXEC) = 6
poll([{fd=4, events=POLLIN}], 1, 10000) = 1 ([{fd=4, revents=POLLIN}])
accept4(4, {sa_family=AF_VSOCK, sa_data="\0\0^c\317O\2\0\0\0\0\0\0\0"}, [16], SOCK_CLOEXEC) = 7
poll([{fd=4, events=POLLIN}], 1, 10000) = 1 ([{fd=4, revents=POLLIN}])
accept4(4, {sa_family=AF_VSOCK, sa_data="\0\0_c\317O\2\0\0\0\0\0\0\0"}, [16], SOCK_CLOEXEC) = 8
poll([{fd=4, events=POLLIN}], 1, 10000) = 1 ([{fd=4, revents=POLLIN}])
accept4(4, {sa_family=AF_VSOCK, sa_data="\0\0`c\317O\2\0\0\0\0\0\0\0"}, [16], SOCK_CLOEXEC) = 9
close(4)
= 0
rt_sigprocmask(SIG_BLOCK, [INT WINCH], NULL, 8) = 0
signalfd4(-1, [INT WINCH], 8, 0) = 4
poll([{fd=0, events=POLLIN}, {fd=7, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLIN}, {fd=4, events=POLLIN}], 5, -1
wsl.exe --version
$ strace wsl.exe --version
execve("/mnt/c/Windows/system32/wsl.exe", ["wsl.exe", "--version"], 0x7fff7408be88 /* 37 vars */) = 0
arch_prctl(ARCH_SET_FS, 0x378290) = 0
set_tid_address(0x3782c8) = 282
brk(NULL) = 0x2100000
brk(0x2101000) = 0x2101000
brk(0x2102000) = 0x2102000
gettid() = 282
gettid() = 282
gettid() = 282
gettid() = 282
brk(0x2103000) = 0x2103000
brk(0x2104000) = 0x2104000
brk(0x2105000) = 0x2105000
sched_getaffinity(0, 128, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) = 32
getpid() = 282
getcwd("/home/tiago", 4096) = 12
uname({sysname="Linux", nodename="DESKTOP-VN589IP", ...}) = 0
getcwd("/home/tiago", 4096) = 12
open("/mnt/c/Windows/system32/wsl.exe", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_PATH) = 3
readlink("/proc/self/fd/3", "/mnt/c/Windows/system32/wsl.exe", 4095) = 31
fstat(3, {st_mode=S_IFREG|0555, st_size=172032, ...}) = 0
stat("/mnt/c/Windows/system32/wsl.exe", {st_mode=S_IFREG|0555, st_size=172032, ...}) = 0
close(3) = 0
open("/proc/self/mountinfo", O_RDONLY) = 3
readv(3, [{iov_base="", iov_len=0}, {iov_base="68 72 0:26 / /mnt/wsl rw,relatim"..., iov_len=1024}], 2) = 1024
readv(3, [{iov_base="", iov_len=0}, {iov_base="- tmpfs none rw,mode=755\n96 95 0"..., iov_len=1024}], 2) = 1024
readv(3, [{iov_base="", iov_len=0}, {iov_base="ev,noexec,relatime - cgroup cgro"..., iov_len=1024}], 2) = 1024
readv(3, [{iov_base="", iov_len=0}, {iov_base="66 / /mnt/c rw,noatime - 9p drvf"..., iov_len=1024}], 2) = 270
readv(3, [{iov_base="", iov_len=0}, {iov_base="", iov_len=1024}], 2) = 0
close(3) = 0
getcwd("/home/tiago", 4096) = 12
open("/proc/self/mountinfo", O_RDONLY) = 3
readv(3, [{iov_base="", iov_len=0}, {iov_base="68 72 0:26 / /mnt/wsl rw,relatim"..., iov_len=1024}], 2) = 1024
readv(3, [{iov_base="", iov_len=0}, {iov_base="- tmpfs none rw,mode=755\n96 95 0"..., iov_len=1024}], 2) = 1024
readv(3, [{iov_base="", iov_len=0}, {iov_base="ev,noexec,relatime - cgroup cgro"..., iov_len=1024}], 2) = 1024
readv(3, [{iov_base="", iov_len=0}, {iov_base="66 / /mnt/c rw,noatime - 9p drvf"..., iov_len=1024}], 2) = 270
readv(3, [{iov_base="", iov_len=0}, {iov_base="", iov_len=1024}], 2) = 0
close(3) = 0
ioctl(0, TCGETS, {B38400 opost isig icanon echo ...}) = 0
ioctl(1, TCGETS, {B38400 opost isig icanon echo ...}) = 0
ioctl(2, TCGETS, {B38400 opost isig icanon echo ...}) = 0
ioctl(0, TIOCGPGRP, [279]) = 0
getpgid(0) = 279
fstat(0, {st_mode=S_IFCHR|0620, st_rdev=makedev(0x88, 0x1), ...}) = 0
fstat(1, {st_mode=S_IFCHR|0620, st_rdev=makedev(0x88, 0x1), ...}) = 0
fstat(2, {st_mode=S_IFCHR|0620, st_rdev=makedev(0x88, 0x1), ...}) = 0
ioctl(0, TIOCGWINSZ, {ws_row=39, ws_col=166, ws_xpixel=0, ws_ypixel=0}) = 0
ioctl(0, SNDCTL_TMR_START or TCSETS, {B38400 -opost -isig -icanon -echo ...}) = 0
dup(0) = 3
socket(AF_VSOCK, SOCK_STREAM|SOCK_CLOEXEC, 0) = 4
bind(4, {sa_family=AF_VSOCK, sa_data="\0\0\377\377\377\377\377\377\377\377\0\0\0\0"}, 16) = 0
getsockname(4, {sa_family=AF_VSOCK, sa_data="\0\0\333\221\264\332\377\377\377\377\0\0\0\0"}, [16]) = 0
listen(4, 4) = 0
socket(AF_UNIX, SOCK_SEQPACKET|SOCK_CLOEXEC, 0) = 5
connect(5, {sa_family=AF_UNIX, sun_path="/run/WSL/244_interop"}, 110) = 0
write(5, "\6\0\0\08\1\0\0\333\221\264\332\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 312) = 312
poll([{fd=4, events=POLLIN}], 1, 10000) = 1 ([{fd=4, revents=POLLIN}])
accept4(4, {sa_family=AF_VSOCK, sa_data="\0\0\32c\317O\2\0\0\0\0\0\0\0"}, [16], SOCK_CLOEXEC) = 6
poll([{fd=4, events=POLLIN}], 1, 10000) = 1 ([{fd=4, revents=POLLIN}])
accept4(4, {sa_family=AF_VSOCK, sa_data="\0\0\33c\317O\2\0\0\0\0\0\0\0"}, [16], SOCK_CLOEXEC) = 7
poll([{fd=4, events=POLLIN}], 1, 10000) = 1 ([{fd=4, revents=POLLIN}])
accept4(4, {sa_family=AF_VSOCK, sa_data="\0\0\34c\317O\2\0\0\0\0\0\0\0"}, [16], SOCK_CLOEXEC) = 8
poll([{fd=4, events=POLLIN}], 1, 10000) = 1 ([{fd=4, revents=POLLIN}])
accept4(4, {sa_family=AF_VSOCK, sa_data="\0\0\35c\317O\2\0\0\0\0\0\0\0"}, [16], SOCK_CLOEXEC) = 9
close(4)
= 0
rt_sigprocmask(SIG_BLOCK, [INT WINCH], NULL, 8) = 0
signalfd4(-1, [INT WINCH], 8, 0) = 4
poll([{fd=0, events=POLLIN}, {fd=7, events=POLLIN}, {fd=8, events=POLLIN}, {fd=9, events=POLLIN}, {fd=4, events=POLLIN}], 5, -1
About this issue
- Original URL
- State: closed
- Created 5 years ago
- Reactions: 16
- Comments: 80 (4 by maintainers)
Thanks a lot to everyone who posted logs and helped with this investigation.
We have found the root cause: A bug in p9rdr.sys (the plan 9 redirector driver), which handles the
\\wsl$\<distro>file accesses. The bug causes a transaction to be left in a pending state forever, which is why this only happens to Windows commands running inside the wsl filesystem (since the programs try to access their current directories, which would in this case be under\\wsl$\<distro>).The fix has been checked in to the Windows codebase, and will work its way through Windows release pipeline. I’ll update the issue when I’ll have the Windows build numbers with the fix.
Update on this issue:
It turns out that this problem is actually composed of 3 bugs:
1) The p9rdr.sys bug I mentioned earlier 2) A bug in WSL’s linux plan9 server that would cause sockets to sometimes be dropped when a new client connects 3) A bug in WSL’s init process that causes error logs from the plan9 server to go to /dev/null instead of dmesg.
Explanation:
1) is what causes the actual ‘hang’, but it can only be triggered if the server closes the socket right after the connection is established (so only happens when 2) happens). And because of 3), there were no records of 2) happening, which ‘hid’ where the issue actually came from.
While 1) is part of the Windows image (and the fix hasn’t shipped to insider builds yet), 2) an 3) have been fixed in the latest store WSL release that we just published today (0.58.0). With those fixes, the code path that triggers 1) shouldn’t be reached anymore, so the issue should go away.
Note that 2) is also present is the inbox version of WSL (although it appears to hit less frequently than with the store version). We have also checked-in fix to the Windows image, but it hasn’t shipped yet.
TL; DR: The fix is available in the latest Store WSL (0.58.0)
Don’t mind me. I’m just one more guy telling Microsoft that there are thousands of developers desperately waiting for the rollout of this fix…
https://youtrack.jetbrains.com/issue/IDEA-276250
@OneBlue will the fix also come in the Insider Dev channel? Can we expect it in the next week build?
Waiting for this one so long!
Thanks for the logs, I’ll start looking at this.
1 month this was fixed in Windows code but it hasn’t been shipped yet…
This workaround is not confirmed to work for everyone, myself included.
same, I had this issue for many months, it made me dualboot linux again
Considering you guys can just use the wsl from the system instead (to workaround the regression) of the store version is it that hard to wait? o.o I mean you can also workaround this by simply using vscode remote extension
Given that the fix is unrelated to wsl, that means the earlier confirmations that rolling back to the non-store version did not work are valid. Please also note; the latest Windows 11 Insider Preview Build 22581 does also not seem to have the fix.
The fix hasn’t shipped yet. I’ll update this thread with the build numbers when it will be in the Dev channel.
@alexbogias Doesn’t that, on the contrary, mean, that your problems are no longer related to this issue?
Yes, your problems are/were likely caused by multiple bugs jumbled together. But have you actually read through the whole https://youtrack.jetbrains.com/issue/IDEA-276250 issue and its comments where you posted? There are clear workarounds mentioned that most likely work for your problems too. And even though there probably is another WSL issue involved, it’s not related to this particular WSL issue, but something along the lines of https://github.com/microsoft/WSL/issues/7252…
One day, I hope the projects that first instigated users to just use the issue tracker in place of a regular user group will go down in history books for the slippery slope they put all of us on…
When will non-preview WSL update? Will we be able to update using wsl --update?
You need to install https://www.microsoft.com/store/productId/9P9TQF7MRM4R which will then install the version, @OneBlue mentioned. Been running for a couple of hours now and I had no issues so far.
I have to say it did work for me 🤷🏻♂️ I was hitting this (or some similar) issue daily. Multiple times. An hour or two (at most) of using some JetBrains IDE or VS Code accessing files via the Plan 9 FS was enough for the \wsl$ mount to become completely unresponsive and unfixable any other way than rebooting.
After I removed the WSL Preview from MS Store altogether and re-installed the Windows Subsystem for Linux “Windows feature”, I haven’t hit the issue once for more than a month now. I’m running a non-insider Windows 11 build (22000.556) though.
@myusrn If you use the Remote - WSL extension,
code .runs a local vscode-server instance on the WSL distro and doesn’t use Plan 9 for file access. Used it all the time when I was hitting this bug…Update: we’re still actively investigating this issue.
We know that the problem is between the Plan9 client on the host and the server that runs in WSL2, but we don’t know what triggers this ‘stuck’ state yet. I’ll update this issue when we’ll have more info.
Managed to fix the encoding error by replacing
UTF-8withUTF-16in theTerminal.wprpfile.logs.zip
@cubic3d Created a new issue on Github with complete WSL logs https://github.com/microsoft/WSL/issues/8824.
默认分发: Ubuntu 默认版本: 2 WSL 版本: 0.61.8.0 内核版本: 5.10.102.1 WSLg 版本: 1.0.39 MSRDC 版本: 1.2.3213 Direct3D 版本: 1.601.0 DXCore 版本: 10.0.25131.1002-220531-1700.rs-onecore-base2-hyp Windows版本: 10.0.22000.739
@OneBlue I reproduced this problem on a new version, is it only fixed on 0.58.0
Phpstorm acts differently now with the latest wsl update, thats why. I think its more than one bug to be honest. And yes it looks like there are also jetbrains bug related 😦
Fyi, i had the windows wsl feature enabled as work around until fix for this came out. After announcement above i reinstalled the windows store wsl preview app then unchecked/removed the windows wsl feature and then rebooted. After that the
wsl --versioncommand produces the following output confirming install of the wsl preview store app that is expected to have the fixes in place. So for file explorer access to\\wsl.localhostand\\wsl$paths has been working without any hangs eventually showing up.Also noticed that wsl preview store app, vs windows wsl feature install option honors windows host mouse button settings in wslg xclient/xserver gui app sessions. For example if you have swapped left and right mouse bottoms in windows host they will be swapped as well in wslg xclient/xserver gui app sessions which is nice level of integration. Also am finding my wsl linux distro wslg supported edge and chrome browser sessions are not hanging sporadically as they did with the windows wsl feature based install.
Any update on when the Plan 9 redirector driver fix will get shiped via Windows Update? Can we manually already install the fix?
@Ansuel I’ve tried it for a few hours and processes aren’t stuck. For this reason I’ve written “it seems”. We need some other confirms to declare this issue permanently (I hope) fixed!
It seems the issue has been fixed in the latest Insider Build (Dev channel)
Thanks to everyone who submitted logs.
We’ve made some progress on the issue, but we need more diagnostics to root cause it.
Instructions once you have a repro (elevated command prompt):
Then using the task manager, check if the process is actually running on the Windows side, if it is, dump it (under the ‘details tab’, right click, then ‘create dump file’). Do the same for all conhost.exe processes running on the machine.
Then reply to this issue with
logs.etland the dumps of all the above processes, if any. Please also include your Windows build number, and the output ofwsl --versionThis happens to me daily when I try to launch VS Code using
code .from WSL. It also happens with one of my VS Code extensions that shells out to Powershell to do manipulate the Windows clipboard. From the linked #7754 thread, this seems to only be happening with the store installed version. Is there a recommended way to revert from the store version to the inbox version? Or should I just uninstall the store version?I’ve found #7754 which sounds exactly like this issue. Maybe it’s a good idea to mark this as duplicate.
Some more info I found about the issue:
When this issue happens, if I run windows commands (e.g.
wsl.exe --version) in a linux path (e.g./home/username) it will hang forever like I described above. But if I run the command inside/mnt/cthen it will work as expected.