more SDL fullscreen state tracking fixes, don't update fullscreen flags on failure to change fullscreen state

src/video/SDL_video.c

@@ -1980,6 +1980,7 @@ SDL_RestoreWindow(SDL_Window * window)
 int
 SDL_SetWindowFullscreen(SDL_Window * window, Uint32 flags)
 {
+    Uint32 oldflags;
     CHECK_WINDOW_MAGIC(window, -1);
 
     flags &= FULLSCREEN_MASK;
@@ -1989,10 +1990,17 @@ SDL_SetWindowFullscreen(SDL_Window * window, Uint32 flags)
     }
 
     /* clear the previous flags and OR in the new ones */
+    oldflags = window->flags & FULLSCREEN_MASK;
     window->flags &= ~FULLSCREEN_MASK;
     window->flags |= flags;
 
-    return SDL_UpdateFullscreenMode(window, FULLSCREEN_VISIBLE(window));
+    if (SDL_UpdateFullscreenMode(window, FULLSCREEN_VISIBLE(window)) == 0) {
+        return 0;
+    }
+    
+    window->flags &= ~FULLSCREEN_MASK;
+    window->flags |= oldflags;
+    return -1;
 }
 
 static SDL_Surface *

src/video/cocoa/SDL_cocoawindow.m

@@ -646,6 +646,8 @@ SetWindowStyle(SDL_Window * window, unsigned int style)
 
     isFullscreenSpace = NO;
     inFullscreenTransition = NO;
+    
+    [self windowDidExitFullScreen:nil];
 }
 
 - (void)windowDidEnterFullScreen:(NSNotification *)aNotification
@@ -690,6 +692,8 @@ SetWindowStyle(SDL_Window * window, unsigned int style)
     
     isFullscreenSpace = YES;
     inFullscreenTransition = NO;
+    
+    [self windowDidEnterFullScreen:nil];
 }
 
 - (void)windowDidExitFullScreen:(NSNotification *)aNotification