Only shadow ARGB windows with a frame outside the frame

An ARGB window with a frame is likely something like a transparent
terminal. It looks awful (and breaks transparency) to draw a big
opaque black shadow under the window, so clip out the region under
the terminal from the shadow we draw.

Add meta_window_get_frame_bounds() to get a cairo region for the
outer bounds of the frame of a window, and modify the frame handling
code to notice changes to the frame shape and discard a cached
region. meta_frames_apply_shapes() is refactored so we can extract
meta_frames_get_frame_bounds() from it.

https://bugzilla.gnome.org/show_bug.cgi?id=635268

NOTE: Applied only partially, compositor part is still missing...

upstream commit:
https://gitlab.gnome.org/GNOME/metacity/commit/0f2e32d1

1 files changed, 119 insertions, 68 deletions

diff --git a/src/ui/frames.c b/src/ui/frames.c
index dfeff3e5..18539e19 100644
--- a/src/ui/frames.c
+++ b/src/ui/frames.c
@@ -839,59 +839,24 @@ apply_cairo_region_to_window (Display        *display,
 }
 #endif
 
-void
-meta_frames_apply_shapes (MetaFrames *frames,
-                          Window      xwindow,
-                          int         new_window_width,
-                          int         new_window_height,
-                          gboolean    window_has_shape)
+static cairo_region_t *
+get_bounds_region (MetaFrames        *frames,
+                   MetaUIFrame       *frame,
+                   MetaFrameGeometry *fgeom,
+                   int                window_width,
+                   int                window_height)
 {
-#ifdef HAVE_SHAPE
-  /* Apply shapes as if window had new_window_width, new_window_height */
-  MetaUIFrame *frame;
-  MetaFrameGeometry fgeom;
-  cairo_rectangle_int_t rect;
   cairo_region_t *corners_region;
-  cairo_region_t *window_region;
+  cairo_region_t *bounds_region;
+  cairo_rectangle_int_t rect;
   gint scale;
 
-  frame = meta_frames_lookup_window (frames, xwindow);
-  g_return_if_fail (frame != NULL);
-
-  meta_frames_calc_geometry (frames, frame, &fgeom);
-
-  if (!(fgeom.top_left_corner_rounded_radius != 0 ||
-        fgeom.top_right_corner_rounded_radius != 0 ||
-        fgeom.bottom_left_corner_rounded_radius != 0 ||
-        fgeom.bottom_right_corner_rounded_radius != 0 ||
-        window_has_shape))
-    {
-      if (frame->shape_applied)
-        {
-          meta_topic (META_DEBUG_SHAPES,
-                      "Unsetting shape mask on frame 0x%lx\n",
-                      frame->xwindow);
-
-          XShapeCombineMask (GDK_DISPLAY_XDISPLAY (gdk_display_get_default ()), frame->xwindow,
-                             ShapeBounding, 0, 0, None, ShapeSet);
-          frame->shape_applied = FALSE;
-        }
-      else
-        {
-          meta_topic (META_DEBUG_SHAPES,
-                      "Frame 0x%lx still doesn't need a shape mask\n",
-                      frame->xwindow);
-        }
-
-      return; /* nothing to do */
-    }
-
   corners_region = cairo_region_create ();
   scale = gdk_window_get_scale_factor (frame->window);
 
-  if (fgeom.top_left_corner_rounded_radius != 0)
+  if (fgeom->top_left_corner_rounded_radius != 0)
     {
-      const int corner = fgeom.top_left_corner_rounded_radius * scale;
+      const int corner = fgeom->top_left_corner_rounded_radius * scale;
       const float radius = sqrt(corner) + corner;
       int i;
 
@@ -907,16 +872,16 @@ meta_frames_apply_shapes (MetaFrames *frames,
         }
     }
 
-  if (fgeom.top_right_corner_rounded_radius != 0)
+  if (fgeom->top_right_corner_rounded_radius != 0)
     {
-      const int corner = fgeom.top_right_corner_rounded_radius * scale;
+      const int corner = fgeom->top_right_corner_rounded_radius * scale;
       const float radius = sqrt(corner) + corner;
       int i;
 
       for (i=0; i<corner; i++)
         {
           const int width = floor(0.5 + radius - sqrt(radius*radius - (radius-(i+0.5))*(radius-(i+0.5))));
-          rect.x = new_window_width - width;
+          rect.x = window_width - width;
           rect.y = i;
           rect.width = width;
           rect.height = 1;
@@ -925,9 +890,9 @@ meta_frames_apply_shapes (MetaFrames *frames,
         }
     }
 
-  if (fgeom.bottom_left_corner_rounded_radius != 0)
+  if (fgeom->bottom_left_corner_rounded_radius != 0)
     {
-      const int corner = fgeom.bottom_left_corner_rounded_radius * scale;
+      const int corner = fgeom->bottom_left_corner_rounded_radius * scale;
       const float radius = sqrt(corner) + corner;
       int i;
 
@@ -935,7 +900,7 @@ meta_frames_apply_shapes (MetaFrames *frames,
         {
           const int width = floor(0.5 + radius - sqrt(radius*radius - (radius-(i+0.5))*(radius-(i+0.5))));
           rect.x = 0;
-          rect.y = new_window_height - i - 1;
+          rect.y = window_height - i - 1;
           rect.width = width;
           rect.height = 1;
 
@@ -943,17 +908,17 @@ meta_frames_apply_shapes (MetaFrames *frames,
         }
     }
 
-  if (fgeom.bottom_right_corner_rounded_radius != 0)
+  if (fgeom->bottom_right_corner_rounded_radius != 0)
     {
-      const int corner = fgeom.bottom_right_corner_rounded_radius * scale;
+      const int corner = fgeom->bottom_right_corner_rounded_radius * scale;
       const float radius = sqrt(corner) + corner;
       int i;
 
       for (i=0; i<corner; i++)
         {
           const int width = floor(0.5 + radius - sqrt(radius*radius - (radius-(i+0.5))*(radius-(i+0.5))));
-          rect.x = new_window_width - width;
-          rect.y = new_window_height - i - 1;
+          rect.x = window_width - width;
+          rect.y = window_height - i - 1;
           rect.width = width;
           rect.height = 1;
 
@@ -961,19 +926,89 @@ meta_frames_apply_shapes (MetaFrames *frames,
         }
     }
 
-  window_region = cairo_region_create ();
+  bounds_region = cairo_region_create ();
 
   rect.x = 0;
   rect.y = 0;
-  rect.width = new_window_width;
-  rect.height = new_window_height;
+  rect.width = window_width;
+  rect.height = window_height;
 
-  cairo_region_union_rectangle (window_region, &rect);
+  cairo_region_union_rectangle (bounds_region, &rect);
 
-  cairo_region_subtract (window_region, corners_region);
+  cairo_region_subtract (bounds_region, corners_region);
 
   cairo_region_destroy (corners_region);
 
+  return bounds_region;
+}
+
+static cairo_region_t *
+get_client_region (MetaFrameGeometry *fgeom,
+                   int                window_width,
+                   int                window_height)
+{
+  cairo_rectangle_int_t rect;
+
+  rect.x = fgeom->left_width;
+  rect.y = fgeom->top_height;
+  rect.width = window_width - fgeom->right_width - rect.x;
+  rect.height = window_height - fgeom->bottom_height - rect.y;
+
+  return cairo_region_create_rectangle (&rect);
+}
+
+void
+meta_frames_apply_shapes (MetaFrames *frames,
+                          Window      xwindow,
+                          int         new_window_width,
+                          int         new_window_height,
+                          gboolean    window_has_shape)
+{
+#ifdef HAVE_SHAPE
+  /* Apply shapes as if window had new_window_width, new_window_height */
+  MetaUIFrame *frame;
+  MetaFrameGeometry fgeom;
+  cairo_region_t *window_region;
+  Display *display;
+
+  frame = meta_frames_lookup_window (frames, xwindow);
+  g_return_if_fail (frame != NULL);
+
+  display = GDK_DISPLAY_XDISPLAY (gdk_display_get_default ());
+
+  meta_frames_calc_geometry (frames, frame, &fgeom);
+
+  if (!(fgeom.top_left_corner_rounded_radius != 0 ||
+        fgeom.top_right_corner_rounded_radius != 0 ||
+        fgeom.bottom_left_corner_rounded_radius != 0 ||
+        fgeom.bottom_right_corner_rounded_radius != 0 ||
+        window_has_shape))
+    {
+      if (frame->shape_applied)
+        {
+          meta_topic (META_DEBUG_SHAPES,
+                      "Unsetting shape mask on frame 0x%lx\n",
+                      frame->xwindow);
+
+          XShapeCombineMask (display, frame->xwindow,
+                             ShapeBounding, 0, 0, None, ShapeSet);
+          frame->shape_applied = FALSE;
+        }
+      else
+        {
+          meta_topic (META_DEBUG_SHAPES,
+                      "Frame 0x%lx still doesn't need a shape mask\n",
+                      frame->xwindow);
+        }
+
+      return; /* nothing to do */
+    }
+
+  window_region = get_bounds_region (frames,
+                                     frame,
+                                     &fgeom,
+                                     new_window_width,
+                                     new_window_height);
   if (window_has_shape)
     {
       /* The client window is oclock or something and has a shape
@@ -1025,14 +1060,9 @@ meta_frames_apply_shapes (MetaFrames *frames,
       /* Punch the client area out of the normal frame shape,
        * then union it with the shape_window's existing shape
        */
-      client_region = cairo_region_create ();
-
-      rect.x = fgeom.left_width;
-      rect.y = fgeom.top_height;
-      rect.width = new_window_width - fgeom.right_width - rect.x;
-      rect.height = new_window_height - fgeom.bottom_height - rect.y;
-
-      cairo_region_union_rectangle (client_region, &rect);
+      client_region = get_client_region (&fgeom,
+                                         new_window_width,
+                                         new_window_height);
 
       cairo_region_subtract (window_region, client_region);
 
@@ -1068,6 +1098,27 @@ meta_frames_apply_shapes (MetaFrames *frames,
 #endif /* HAVE_SHAPE */
 }
 
+cairo_region_t *
+meta_frames_get_frame_bounds (MetaFrames *frames,
+                              Window      xwindow,
+                              int         window_width,
+                              int         window_height)
+{
+  MetaUIFrame *frame;
+  MetaFrameGeometry fgeom;
+
+  frame = meta_frames_lookup_window (frames, xwindow);
+  g_return_val_if_fail (frame != NULL, NULL);
+
+  meta_frames_calc_geometry (frames, frame, &fgeom);
+
+  return get_bounds_region (frames,
+                            frame,
+                            &fgeom,
+                            window_width,
+                            window_height);
+}
+
 void
 meta_frames_move_resize_frame (MetaFrames *frames,
                                Window      xwindow,