wayland: Rework aspect ratio correction

Try harder to keep a valid aspect ratio, considering the window size limits, and in cases where the window size limits prevent achieving the desired aspect ratio, the window size limits should win overall.

src/video/wayland/SDL_waylandwindow.c

@@ -398,7 +398,7 @@ static void ConfigureWindowGeometry(SDL_Window *window)
             viewport_height = data->requested.pixel_height;
         }
 
-        if (data->viewport && data->waylandData->subcompositor && !data->is_fullscreen) {
+        if (data->viewport && data->waylandData->subcompositor && !data->floating && !data->is_fullscreen) {
             if (window->min_w) {
                 viewport_width = SDL_max(viewport_width, window->min_w);
             }
@@ -915,6 +915,124 @@ static const struct xdg_surface_listener _xdg_surface_listener = {
     handle_xdg_surface_configure
 };
 
+static void ApplyGeometryLimits(SDL_Window *window, int req_w, int req_h, int *out_w, int *out_h)
+{
+    SDL_WindowData *wind = window->internal;
+
+    int w = req_w;
+    int h = req_h;
+    const float aspect = (float)req_w / (float)req_h;
+
+    /* If adjusting for aspect causes the adjusted dimension to exceed the min/max limit, clamp to the limit
+     * and attempt to adjust the opposite dimension to compensate.
+     */
+    if (!wind->resizing || (wind->resize_edge & WAYLAND_RESIZE_EDGE_CORNER) == WAYLAND_RESIZE_EDGE_CORNER) {
+        if (window->min_aspect != 0.f && aspect < window->min_aspect) {
+            h = SDL_max(SDL_lroundf((float)req_w / window->min_aspect), 1);
+
+            if (h < window->min_h) {
+                w = SDL_max(SDL_lroundf((float)window->min_h * window->min_aspect), 1);
+            }
+            if (window->max_h && h > window->max_h) {
+                w = SDL_max(SDL_lroundf((float)window->max_h * window->min_aspect), 1);
+            }
+        } else if (window->max_aspect != 0.f && aspect > window->max_aspect) {
+            w = SDL_max(SDL_lroundf((float)req_h * window->max_aspect), 1);
+
+            if (w < window->min_w) {
+                h = SDL_max(SDL_lroundf((float)window->min_w / window->max_aspect), 1);
+            }
+            if (window->max_w && w > window->max_w) {
+                h = SDL_max(SDL_lroundf((float)window->max_w / window->max_aspect), 1);
+            }
+        }
+    } else if (wind->resize_edge & WAYLAND_RESIZE_EDGE_LR) {
+        if (window->min_aspect != 0.f && aspect < window->min_aspect) {
+            h = SDL_max(SDL_lroundf((float)req_w / window->min_aspect), 1);
+
+            if (h < window->min_h) {
+                w = SDL_max(SDL_lroundf((float)window->min_h * window->min_aspect), 1);
+            }
+            if (window->max_h && h > window->max_h) {
+                w = SDL_max(SDL_lroundf((float)window->max_h * window->min_aspect), 1);
+            }
+        } else if (window->max_aspect != 0.f && aspect > window->max_aspect) {
+            h = SDL_max(SDL_lroundf((float)req_w / window->max_aspect), 1);
+
+            if (h < window->min_h) {
+                w = SDL_max(SDL_lroundf((float)window->min_h * window->max_aspect), 1);
+            }
+            if (window->max_h && h > window->max_h) {
+                w = SDL_max(SDL_lroundf((float)window->max_h * window->max_aspect), 1);
+            }
+        }
+    } else if (wind->resize_edge & WAYLAND_RESIZE_EDGE_TB) {
+        if (window->min_aspect != 0.f && aspect < window->min_aspect) {
+            w = SDL_max(SDL_lroundf((float)req_h * window->min_aspect), 1);
+
+            if (w < window->min_w) {
+                h = SDL_max(SDL_lroundf((float)window->min_w / window->min_aspect), 1);
+            }
+            if (window->max_w && w > window->max_w) {
+                h = SDL_max(SDL_lroundf((float)window->max_w / window->min_aspect), 1);
+            }
+        } else if (window->max_aspect != 0.f && aspect > window->max_aspect) {
+            w = SDL_max(SDL_lroundf((float)req_h * window->max_aspect), 1);
+
+            if (w < window->min_w) {
+                h = SDL_max(SDL_lroundf((float)window->min_w / window->max_aspect), 1);
+            }
+            if (window->max_w && w > window->max_w) {
+                h = SDL_max(SDL_lroundf((float)window->max_w / window->max_aspect), 1);
+            }
+        }
+    }
+
+    if (window->max_w) {
+        w = SDL_min(w, window->max_w);
+    }
+    *out_w = SDL_max(w, window->min_w);
+
+    if (window->max_h) {
+        h = SDL_min(h, window->max_h);
+    }
+    *out_h = SDL_max(h, window->min_h);
+
+}
+
+static void DetermineResizeAxis(SDL_WindowData *wind, int width, int height, bool resizing)
+{
+    /* Try to determine the axis along which the window is being resized.
+     *
+     * Every compositor is different, and some will start sending strange values once the resize
+     * begins, so only the first change is safe to try and determine the resize edge.
+     *
+     * Some compositors don't send a configure event with the resize flag cleared when the resize ends,
+     * so a reset timer is required.
+     */
+    if (resizing) {
+        const Uint64 now = SDL_GetTicksNS();
+        if (now >= wind->last_resize_event_time_ns + SDL_MS_TO_NS(500)) {
+            wind->resize_edge = 0;
+        }
+
+        wind->last_resize_event_time_ns = now;
+
+        if (!wind->resize_edge) {
+            if (width != wind->last_configure.width || !height) {
+                wind->resize_edge |= WAYLAND_RESIZE_EDGE_LR;
+            }
+            if (height != wind->last_configure.height || !width) {
+                wind->resize_edge |= WAYLAND_RESIZE_EDGE_TB;
+            }
+        }
+    } else {
+        wind->resize_edge = 0;
+    }
+
+    wind->resizing = resizing;
+}
+
 static void handle_xdg_toplevel_configure(void *data,
                                           struct xdg_toplevel *xdg_toplevel,
                                           int32_t width,
@@ -979,6 +1097,7 @@ static void handle_xdg_toplevel_configure(void *data,
     // When resizing, dimensions other than 0 are a maximum.
     const bool new_configure_size = width != wind->last_configure.width || height != wind->last_configure.height;
 
+    DetermineResizeAxis(wind, width, height, resizing);
     UpdateWindowFullscreen(window, fullscreen);
 
     /* Always send a maximized/restore event; if the event is redundant it will
@@ -1090,54 +1209,15 @@ static void handle_xdg_toplevel_configure(void *data,
          * - Only floating windows are truly safe to resize: maximized windows must have
          *   their exact dimensions respected, or a protocol violation can occur, and tiled
          *   windows can technically use dimensions smaller than the ones supplied by the
-         *   compositor, but doing so can cause odd behavior. In these cases it's best to use
-         *   the supplied dimensions and use a viewport + mask to enforce the size limits and/or
-         *   aspect ratio.
-         *
-         * - When resizing a window, the width/height are maximum values, so aspect ratio
-         *   correction can't resize beyond the existing dimensions, or a protocol violation
-         *   can occur. However, in practice, nothing seems to kill clients that do this, but
-         *   doing so can cause certain compositors to glitch out.
+         *   compositor, but doing so can cause visual glitches and odd behavior. In these cases
+         *   it's best to use the supplied dimensions and use a viewport + mask to enforce the
+         *   size limits and/or aspect ratio.
          */
         if (floating) {
             if (!wind->scale_to_display) {
-                if (window->max_w > 0) {
-                    wind->requested.logical_width = SDL_min(wind->requested.logical_width, window->max_w);
-                }
-                wind->requested.logical_width = SDL_max(wind->requested.logical_width, window->min_w);
-
-                if (window->max_h > 0) {
-                    wind->requested.logical_height = SDL_min(wind->requested.logical_height, window->max_h);
-                }
-                wind->requested.logical_height = SDL_max(wind->requested.logical_height, window->min_h);
-
-                // Aspect correction.
-                const float aspect = (float)wind->requested.logical_width / (float)wind->requested.logical_height;
-
-                if (window->min_aspect != 0.f && aspect < window->min_aspect) {
-                    wind->requested.logical_height = SDL_max(SDL_lroundf((float)wind->requested.logical_width / window->min_aspect), 1);
-                } else if (window->max_aspect != 0.f && aspect > window->max_aspect) {
-                    wind->requested.logical_width = SDL_max(SDL_lroundf((float)wind->requested.logical_height * window->max_aspect), 1);
-                }
+                ApplyGeometryLimits(window, wind->requested.logical_width, wind->requested.logical_height, &wind->requested.logical_width, &wind->requested.logical_height);
             } else {
-                if (window->max_w > 0) {
-                    wind->requested.pixel_width = SDL_min(wind->requested.pixel_width, window->max_w);
-                }
-                wind->requested.pixel_width = SDL_max(wind->requested.pixel_width, window->min_w);
-
-                if (window->max_h > 0) {
-                    wind->requested.pixel_height = SDL_min(wind->requested.pixel_height, window->max_h);
-                }
-                wind->requested.pixel_height = SDL_max(wind->requested.pixel_height, window->min_h);
-
-                // Aspect correction.
-                const float aspect = (float)wind->requested.pixel_width / (float)wind->requested.pixel_height;
-
-                if (window->min_aspect != 0.f && aspect < window->min_aspect) {
-                    wind->requested.pixel_height = SDL_max(SDL_lroundf((float)wind->requested.pixel_width / window->min_aspect), 1);
-                } else if (window->max_aspect != 0.f && aspect > window->max_aspect) {
-                    wind->requested.pixel_width = SDL_max(SDL_lroundf((float)wind->requested.pixel_height * window->max_aspect), 1);
-                }
+                ApplyGeometryLimits(window, wind->requested.pixel_width, wind->requested.pixel_height, &wind->requested.pixel_width, &wind->requested.pixel_height);
 
                 wind->requested.logical_width = PixelToPoint(window, wind->requested.pixel_width);
                 wind->requested.logical_height = PixelToPoint(window, wind->requested.pixel_height);
@@ -1165,7 +1245,6 @@ static void handle_xdg_toplevel_configure(void *data,
     wind->suspended = suspended;
     wind->active = active;
     window->tiled = tiled;
-    wind->resizing = resizing;
 }
 
 static void handle_xdg_toplevel_close(void *data, struct xdg_toplevel *xdg_toplevel)
@@ -1487,6 +1566,7 @@ static void decoration_frame_configure(struct libdecor_frame *frame,
             }
 
             const bool new_configure_size = width != wind->last_configure.width || height != wind->last_configure.height;
+            DetermineResizeAxis(wind, width, height, resizing);
 
             if (!width) {
                 /* This happens when we're being restored from a non-floating state,
@@ -1580,51 +1660,12 @@ static void decoration_frame_configure(struct libdecor_frame *frame,
          *   compositor, but doing so can cause odd behavior. In these cases it's best to use
          *   the supplied dimensions and use a viewport + mask to enforce the size limits and/or
          *   aspect ratio.
-         *
-         * - When resizing a window, the width/height are maximum values, so aspect ratio
-         *   correction can't resize beyond the existing dimensions, or a protocol violation
-         *   can occur. However, in practice, nothing seems to kill clients that do this, but
-         *   doing so can cause certain compositors to glitch out.
          */
         if (floating) {
             if (!wind->scale_to_display) {
-                if (window->max_w > 0) {
-                    wind->requested.logical_width = SDL_min(wind->requested.logical_width, window->max_w);
-                }
-                wind->requested.logical_width = SDL_max(wind->requested.logical_width, window->min_w);
-
-                if (window->max_h > 0) {
-                    wind->requested.logical_height = SDL_min(wind->requested.logical_height, window->max_h);
-                }
-                wind->requested.logical_height = SDL_max(wind->requested.logical_height, window->min_h);
-
-                // Aspect correction.
-                const float aspect = (float)wind->requested.logical_width / (float)wind->requested.logical_height;
-
-                if (window->min_aspect != 0.f && aspect < window->min_aspect) {
-                    wind->requested.logical_height = SDL_max(SDL_lroundf((float)wind->requested.logical_width / window->min_aspect), 1);
-                } else if (window->max_aspect != 0.f && aspect > window->max_aspect) {
-                    wind->requested.logical_width = SDL_max(SDL_lroundf((float)wind->requested.logical_height * window->max_aspect), 1);
-                }
+                ApplyGeometryLimits(window, wind->requested.logical_width, wind->requested.logical_height, &wind->requested.logical_width, &wind->requested.logical_height);
             } else {
-                if (window->max_w > 0) {
-                    wind->requested.pixel_width = SDL_min(wind->requested.pixel_width, window->max_w);
-                }
-                wind->requested.pixel_width = SDL_max(wind->requested.pixel_width, window->min_w);
-
-                if (window->max_h > 0) {
-                    wind->requested.pixel_height = SDL_min(wind->requested.pixel_height, window->max_h);
-                }
-                wind->requested.pixel_height = SDL_max(wind->requested.pixel_height, window->min_h);
-
-                // Aspect correction.
-                const float aspect = (float)wind->requested.pixel_width / (float)wind->requested.pixel_height;
-
-                if (window->min_aspect != 0.f && aspect < window->min_aspect) {
-                    wind->requested.pixel_height = SDL_max(SDL_lroundf((float)wind->requested.pixel_width / window->min_aspect), 1);
-                } else if (window->max_aspect != 0.f && aspect > window->max_aspect) {
-                    wind->requested.pixel_width = SDL_max(SDL_lroundf((float)wind->requested.pixel_height * window->max_aspect), 1);
-                }
+                ApplyGeometryLimits(window, wind->requested.pixel_width, wind->requested.pixel_height, &wind->requested.pixel_width, &wind->requested.pixel_height);
 
                 wind->requested.logical_width = PixelToPoint(window, wind->requested.pixel_width);
                 wind->requested.logical_height = PixelToPoint(window, wind->requested.pixel_height);

src/video/wayland/SDL_waylandwindow.h

@@ -103,6 +103,13 @@ struct SDL_WindowData
         WAYLAND_TOPLEVEL_CONSTRAINED_BOTTOM = 0x08
     } toplevel_constraints;
 
+    enum
+    {
+        WAYLAND_RESIZE_EDGE_LR = 0x01,
+        WAYLAND_RESIZE_EDGE_TB = 0x02,
+        WAYLAND_RESIZE_EDGE_CORNER = WAYLAND_RESIZE_EDGE_LR | WAYLAND_RESIZE_EDGE_TB,
+    } resize_edge;
+
     struct wl_egl_window *egl_window;
 #ifdef SDL_VIDEO_OPENGL_EGL
     EGLSurface egl_surface;
@@ -218,6 +225,7 @@ struct SDL_WindowData
     SDL_DisplayID last_displayID;
     int pending_state_deadline_count;
     Uint64 last_focus_event_time_ns;
+    Uint64 last_resize_event_time_ns;
     int icc_fd;
     Uint32 icc_size;
     bool floating;