SDL: Exiting fullscreen on X11 breaks rendering

Bug filed with Gnome: https://gitlab.gnome.org/GNOME/gnome-shell/-/issues/5878

If they come back and say “do this instead and that won’t happen,” I’ll reopen this bug to make changes in SDL itself.

Quick followup: this was fixed in Gnome’s revision control a few weeks ago, so expect distros to eventually ship this fix.

https://gitlab.gnome.org/GNOME/mutter/-/commit/46fc94b67f3c1255f5c6be1a33ff8f80d90a0958

Trying to make a small reproduction case for a Gnome bug report…

This is the simplest X11 program I could write to bypass the compositor and toggle fullscreen. Each keypress toggles to/from fullscreen (after six presses it quits).

This does NOT trigger the bug for me, but this is also way simpler than what SDL does.

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#include <X11/Xlib.h>
#include <X11/Xutil.h>
#include <X11/Xatom.h>

#define WINX 100
#define WINY 100
#define WINW 640
#define WINH 480

#define _NET_WM_STATE_REMOVE    0l
#define _NET_WM_STATE_ADD       1l

static Bool isAnyEvent(Display *display, XEvent *ev, XPointer arg) { return True; }

static void set_fullscreen(Display *display, const int screen, Window w, const int fs)
{
    Atom _NET_WM_STATE = XInternAtom(display, "_NET_WM_STATE", False);
    Atom _NET_WM_STATE_FULLSCREEN = XInternAtom(display, "_NET_WM_STATE_FULLSCREEN", False);
    XEvent xevent;

    XSizeHints *sizehints = XAllocSizeHints();
    long flags = 0;
    XGetWMNormalHints(display, w, sizehints, &flags);

    if (fs) {
        /* we are going fullscreen so turn the flags off */
        sizehints->flags &= ~(PMinSize | PMaxSize);
    } else {
        /* Reset the min/max width height to make the window non-resizable again */
        sizehints->flags |= PMinSize | PMaxSize;
        sizehints->min_width = sizehints->max_width = WINW;
        sizehints->min_height = sizehints->max_height = WINH;
    }
    XSetWMNormalHints(display, w, sizehints);
    XFree(sizehints);

    memset(&xevent, 0, sizeof (xevent));
    xevent.xany.type = ClientMessage;
    xevent.xclient.message_type = _NET_WM_STATE;
    xevent.xclient.format = 32;
    xevent.xclient.window = w;
    xevent.xclient.data.l[0] = fs ? _NET_WM_STATE_ADD : _NET_WM_STATE_REMOVE;
    xevent.xclient.data.l[1] = _NET_WM_STATE_FULLSCREEN;
    xevent.xclient.data.l[3] = 0l;
    XSendEvent(display, RootWindow(display, screen), 0, SubstructureNotifyMask | SubstructureRedirectMask, &xevent);
    XFlush(display);
}

int main(int argc, char **argv)
{
    XInitThreads();
    Display *display = XOpenDisplay(NULL);
    const int screen = DefaultScreen(display);
    XSetWindowAttributes xattr;
    XSizeHints *sizehints;
    XWMHints *wmhints;
    XClassHint *classhints;
    Window w;

    memset(&xattr, '\0', sizeof (xattr));
    xattr.event_mask = ExposureMask |
                       ButtonPressMask | ButtonReleaseMask | KeyPressMask | KeyReleaseMask |
                       StructureNotifyMask | FocusChangeMask | PointerMotionMask;

    w = XCreateWindow(display, RootWindow(display, screen),
                       WINX, WINY, WINW, WINH,
                       0, CopyFromParent, InputOutput, CopyFromParent,
                       CWEventMask, &xattr );

    sizehints = XAllocSizeHints();
    sizehints->flags = 0;
    sizehints->min_width = sizehints->max_width = WINW;
    sizehints->min_height = sizehints->max_height = WINH;
    sizehints->flags |= (PMaxSize | PMinSize);

    sizehints->x = WINX;
    sizehints->y = WINY;
    sizehints->flags |= USPosition;

    /* Setup the input hints so we get keyboard input */
    wmhints = XAllocWMHints();
    wmhints->input = True;
    wmhints->window_group = (XID) getpid();
    wmhints->flags = InputHint | WindowGroupHint;

    /* Setup the class hints so we can get an icon (AfterStep) */
    classhints = XAllocClassHint();
    classhints->res_name = "SDL_App";
    classhints->res_class = "SDL_App";

    /* Set the size, input and class hints, and define WM_CLIENT_MACHINE and WM_LOCALE_NAME */
    XSetWMProperties(display, w, NULL, NULL, NULL, 0, sizehints, wmhints, classhints);

    XFree(sizehints);
    XFree(wmhints);
    XFree(classhints);

    long pid = (long) getpid();
    Atom _NET_WM_PID = XInternAtom(display, "_NET_WM_PID", False);
    XChangeProperty(display, w, _NET_WM_PID, XA_CARDINAL, 32, PropModeReplace, (unsigned char *) &pid, 1);

    Atom _NET_WM_WINDOW_TYPE = XInternAtom(display, "_NET_WM_WINDOW_TYPE", False);
    Atom _NET_WM_WINDOW_TYPE_NORMAL = XInternAtom(display, "_NET_WM_WINDOW_TYPE_NORMAL", False);
    XChangeProperty(display, w, _NET_WM_WINDOW_TYPE, XA_ATOM, 32, PropModeReplace, (unsigned char *)&_NET_WM_WINDOW_TYPE_NORMAL, 1);

    long compositor = 1;
    Atom _NET_WM_BYPASS_COMPOSITOR = XInternAtom(display, "_NET_WM_BYPASS_COMPOSITOR", False);
    XChangeProperty(display, w, _NET_WM_BYPASS_COMPOSITOR, XA_CARDINAL, 32, PropModeReplace, (unsigned char *)&compositor, 1);

    XMapRaised(display, w);

    XFlush(display);

    XGCValues ctx_vals;
    const unsigned long gcflags = GCForeground | GCBackground;
    memset(&ctx_vals, 0, sizeof (ctx_vals));
    GC gc = XCreateGC(display, w, gcflags, &ctx_vals);

    int presses = 0;
    while (presses < 6) {
        XEvent xevent;
        while (XCheckIfEvent(display, &xevent, isAnyEvent, NULL)) {
            switch (xevent.type) {
                case KeyPress:
                    printf("KEYPRESS\n");
                    presses++;
                    set_fullscreen(display, screen, w, presses & 1);
                    break;
            }
        }

        XFillRectangle(display, w, gc, 0, 0, 999999, 999999);
        XFlush(display);
        usleep(10000);
    }

    XFreeGC(display, gc);
    XDestroyWindow(display, w);
    XCloseDisplay(display);

    return 0;
}

Confirmed here on Ubuntu 22.04, gnome on x11. I’ll bisect in the morning.