Only shadow ARGB windows with a frame outside the frame

An ARGB window with a frame is likely something like a transparent
terminal. It looks awful (and breaks transparency) to draw a big
opaque black shadow under the window, so clip out the region under
the terminal from the shadow we draw.

Add meta_window_get_frame_bounds() to get a cairo region for the
outer bounds of the frame of a window, and modify the frame handling
code to notice changes to the frame shape and discard a cached
region. meta_frames_apply_shapes() is refactored so we can extract
meta_frames_get_frame_bounds() from it.

https://bugzilla.gnome.org/show_bug.cgi?id=635268

NOTE: Applied only partially, compositor part is still missing...

upstream commit:
https://gitlab.gnome.org/GNOME/metacity/commit/0f2e32d1

9 files changed, 203 insertions, 79 deletions

diff --git a/src/core/frame-private.h b/src/core/frame-private.h
index 1b995f58..d7183149 100644
--- a/src/core/frame-private.h
+++ b/src/core/frame-private.h
@@ -59,10 +59,11 @@ void     meta_frame_queue_draw              (MetaFrame  *frame);
 
 MetaFrameFlags meta_frame_get_flags (MetaFrame *frame);
 
-void meta_frame_sync_to_window    (MetaFrame        *frame,
+gboolean meta_frame_sync_to_window (MetaFrame         *frame,
                                    int               gravity,
                                    gboolean          need_move,
                                    gboolean          need_resize);
+cairo_region_t *meta_frame_get_frame_bounds (MetaFrame *frame);
 
 void meta_frame_set_screen_cursor (MetaFrame        *frame,
                                    MetaCursor        cursor);
diff --git a/src/core/frame.c b/src/core/frame.c
index 25f83ceb..d2f5cfd3 100644
--- a/src/core/frame.c
+++ b/src/core/frame.c
@@ -316,7 +316,7 @@ meta_frame_calc_borders (MetaFrame        *frame,
                              borders);
 }
 
-static void
+static gboolean
 update_shape (MetaFrame *frame)
 {
   if (frame->need_reapply_frame_shape)
@@ -327,10 +327,13 @@ update_shape (MetaFrame *frame)
                                  frame->rect.height,
                                  frame->window->has_shape);
       frame->need_reapply_frame_shape = FALSE;
+      return TRUE;
     }
+  else
+    return FALSE;
 }
 
-void
+gboolean
 meta_frame_sync_to_window (MetaFrame *frame,
                            int        resize_gravity,
                            gboolean   need_move,
@@ -338,8 +341,7 @@ meta_frame_sync_to_window (MetaFrame *frame,
 {
   if (!(need_move || need_resize))
     {
-      update_shape (frame);
-      return;
+      return update_shape (frame);
     }
 
   meta_topic (META_DEBUG_GEOMETRY,
@@ -381,6 +383,16 @@ meta_frame_sync_to_window (MetaFrame *frame,
         meta_ui_repaint_frame (frame->window->screen->ui,
                                frame->xwindow);
     }
+  return need_resize;
+}
+
+cairo_region_t *
+meta_frame_get_frame_bounds (MetaFrame *frame)
+{
+  return meta_ui_get_frame_bounds (frame->window->screen->ui,
+                                   frame->xwindow,
+                                   frame->rect.width,
+                                   frame->rect.height);
 }
 
 void
diff --git a/src/core/window-private.h b/src/core/window-private.h
index 38184231..50bc57be 100644
--- a/src/core/window-private.h
+++ b/src/core/window-private.h
@@ -41,6 +41,7 @@
 #include "stack.h"
 #include "iconcache.h"
 #include <X11/Xutil.h>
+#include <cairo.h>
 #include <gdk-pixbuf/gdk-pixbuf.h>
 #include <gtk/gtk.h>
 
@@ -324,6 +325,9 @@ struct _MetaWindow
   /* if TRUE, application is buggy and SYNC resizing is turned off */
   guint disable_sync : 1;
 
+  /* if non-NULL, the bounds of the window frame */
+  cairo_region_t *frame_bounds;
+
   /* Note: can be NULL */
   GSList *struts;
 
diff --git a/src/core/window.c b/src/core/window.c
index fded7b8e..d6566de3 100644
--- a/src/core/window.c
+++ b/src/core/window.c
@@ -524,6 +524,7 @@ meta_window_new_with_attrs (MetaDisplay       *display,
   window->shaken_loose = FALSE;
   window->have_focus_click_grab = FALSE;
   window->disable_sync = FALSE;
+  window->frame_bounds = NULL;
 
   window->unmaps_pending = 0;
 
@@ -1221,6 +1222,9 @@ meta_window_free (MetaWindow  *window,
   if (window->mini_icon)
     g_object_unref (G_OBJECT (window->mini_icon));
 
+  if (window->frame_bounds)
+    cairo_region_destroy (window->frame_bounds);
+
   meta_icon_cache_free (&window->icon_cache);
 
   g_free (window->sm_client_id);
@@ -3483,6 +3487,7 @@ meta_window_move_resize_internal (MetaWindow          *window,
   int frame_size_dy;
   int size_dx;
   int size_dy;
+  gboolean frame_shape_changed = FALSE;
   gboolean is_configure_request;
   gboolean do_gravity_adjust;
   gboolean is_user_action;
@@ -3789,9 +3794,10 @@ meta_window_move_resize_internal (MetaWindow          *window,
     meta_window_set_gravity (window, StaticGravity);
 
   if (configure_frame_first && have_window_frame)
-    meta_frame_sync_to_window (window->frame,
-                               gravity,
-                               need_move_frame, need_resize_frame);
+    frame_shape_changed = meta_frame_sync_to_window (window->frame,
+                                                     gravity,
+                                                     need_move_frame,
+                                                     need_resize_frame);
 
   values.border_width = 0;
   values.x = client_move_x;
@@ -3846,9 +3852,10 @@ meta_window_move_resize_internal (MetaWindow          *window,
     }
 
   if (!configure_frame_first && have_window_frame)
-    meta_frame_sync_to_window (window->frame,
-                               gravity,
-                               need_move_frame, need_resize_frame);
+    frame_shape_changed = meta_frame_sync_to_window (window->frame,
+                                                     gravity,
+                                                     need_move_frame,
+                                                     need_resize_frame);
 
   /* Put gravity back to be nice to lesser window managers */
   if (use_static_gravity)
@@ -3888,6 +3895,11 @@ meta_window_move_resize_internal (MetaWindow          *window,
    *      server-side size/pos of window->xwindow and frame->xwindow
    *   b) all constraints are obeyed by window->rect and frame->rect
    */
+  if (frame_shape_changed && window->frame_bounds)
+    {
+      cairo_region_destroy (window->frame_bounds);
+      window->frame_bounds = NULL;
+    }
 
   if (meta_prefs_get_attach_modal_dialogs ())
     meta_window_foreach_transient (window, move_attached_dialog, NULL);
@@ -8764,3 +8776,24 @@ meta_window_is_client_decorated (MetaWindow *window)
    */
   return window->has_custom_frame_extents;
 }
+
+/**
+ * meta_window_get_frame_bounds:
+ *
+ * Gets a region representing the outer bounds of the window's frame.
+ *
+ * Return value: (transfer none) (allow-none): a #cairo_region_t
+ * holding the outer bounds of the window, or %NULL if the window
+ * doesn't have a frame.
+ */
+cairo_region_t *
+meta_window_get_frame_bounds (MetaWindow *window)
+{
+  if (!window->frame_bounds)
+    {
+      if (window->frame)
+        window->frame_bounds = meta_frame_get_frame_bounds (window->frame);
+    }
+
+  return window->frame_bounds;
+}
diff --git a/src/include/ui.h b/src/include/ui.h
index 652e8cfc..40ae03cf 100644
--- a/src/include/ui.h
+++ b/src/include/ui.h
@@ -97,6 +97,11 @@ void meta_ui_apply_frame_shape  (MetaUI  *ui,
                                  int      new_window_height,
                                  gboolean window_has_shape);
 
+cairo_region_t *meta_ui_get_frame_bounds (MetaUI *ui,
+                                          Window  xwindow,
+                                          int     window_width,
+                                          int     window_height);
+
 void meta_ui_queue_frame_draw (MetaUI *ui,
                                Window xwindow);
 
diff --git a/src/include/window.h b/src/include/window.h
index 3fd0ee53..0aa208e4 100644
--- a/src/include/window.h
+++ b/src/include/window.h
@@ -23,6 +23,7 @@
 #define META_WINDOW_H
 
 #include <glib.h>
+#include <cairo.h>
 #include <X11/Xlib.h>
 
 #include "boxes.h"
@@ -37,5 +38,6 @@ MetaDisplay *meta_window_get_display (MetaWindow *window);
 Window meta_window_get_xwindow (MetaWindow *window);
 MetaWindow *meta_window_get_transient_for (MetaWindow *window);
 gboolean meta_window_is_maximized (MetaWindow *window);
+cairo_region_t *meta_window_get_frame_bounds (MetaWindow *window);
 
 #endif
diff --git a/src/ui/frames.c b/src/ui/frames.c
index dfeff3e5..18539e19 100644
--- a/src/ui/frames.c
+++ b/src/ui/frames.c
@@ -839,59 +839,24 @@ apply_cairo_region_to_window (Display        *display,
 }
 #endif
 
-void
-meta_frames_apply_shapes (MetaFrames *frames,
-                          Window      xwindow,
-                          int         new_window_width,
-                          int         new_window_height,
-                          gboolean    window_has_shape)
+static cairo_region_t *
+get_bounds_region (MetaFrames        *frames,
+                   MetaUIFrame       *frame,
+                   MetaFrameGeometry *fgeom,
+                   int                window_width,
+                   int                window_height)
 {
-#ifdef HAVE_SHAPE
-  /* Apply shapes as if window had new_window_width, new_window_height */
-  MetaUIFrame *frame;
-  MetaFrameGeometry fgeom;
-  cairo_rectangle_int_t rect;
   cairo_region_t *corners_region;
-  cairo_region_t *window_region;
+  cairo_region_t *bounds_region;
+  cairo_rectangle_int_t rect;
   gint scale;
 
-  frame = meta_frames_lookup_window (frames, xwindow);
-  g_return_if_fail (frame != NULL);
-
-  meta_frames_calc_geometry (frames, frame, &fgeom);
-
-  if (!(fgeom.top_left_corner_rounded_radius != 0 ||
-        fgeom.top_right_corner_rounded_radius != 0 ||
-        fgeom.bottom_left_corner_rounded_radius != 0 ||
-        fgeom.bottom_right_corner_rounded_radius != 0 ||
-        window_has_shape))
-    {
-      if (frame->shape_applied)
-        {
-          meta_topic (META_DEBUG_SHAPES,
-                      "Unsetting shape mask on frame 0x%lx\n",
-                      frame->xwindow);
-
-          XShapeCombineMask (GDK_DISPLAY_XDISPLAY (gdk_display_get_default ()), frame->xwindow,
-                             ShapeBounding, 0, 0, None, ShapeSet);
-          frame->shape_applied = FALSE;
-        }
-      else
-        {
-          meta_topic (META_DEBUG_SHAPES,
-                      "Frame 0x%lx still doesn't need a shape mask\n",
-                      frame->xwindow);
-        }
-
-      return; /* nothing to do */
-    }
-
   corners_region = cairo_region_create ();
   scale = gdk_window_get_scale_factor (frame->window);
 
-  if (fgeom.top_left_corner_rounded_radius != 0)
+  if (fgeom->top_left_corner_rounded_radius != 0)
     {
-      const int corner = fgeom.top_left_corner_rounded_radius * scale;
+      const int corner = fgeom->top_left_corner_rounded_radius * scale;
       const float radius = sqrt(corner) + corner;
       int i;
 
@@ -907,16 +872,16 @@ meta_frames_apply_shapes (MetaFrames *frames,
         }
     }
 
-  if (fgeom.top_right_corner_rounded_radius != 0)
+  if (fgeom->top_right_corner_rounded_radius != 0)
     {
-      const int corner = fgeom.top_right_corner_rounded_radius * scale;
+      const int corner = fgeom->top_right_corner_rounded_radius * scale;
       const float radius = sqrt(corner) + corner;
       int i;
 
       for (i=0; i<corner; i++)
         {
           const int width = floor(0.5 + radius - sqrt(radius*radius - (radius-(i+0.5))*(radius-(i+0.5))));
-          rect.x = new_window_width - width;
+          rect.x = window_width - width;
           rect.y = i;
           rect.width = width;
           rect.height = 1;
@@ -925,9 +890,9 @@ meta_frames_apply_shapes (MetaFrames *frames,
         }
     }
 
-  if (fgeom.bottom_left_corner_rounded_radius != 0)
+  if (fgeom->bottom_left_corner_rounded_radius != 0)
     {
-      const int corner = fgeom.bottom_left_corner_rounded_radius * scale;
+      const int corner = fgeom->bottom_left_corner_rounded_radius * scale;
       const float radius = sqrt(corner) + corner;
       int i;
 
@@ -935,7 +900,7 @@ meta_frames_apply_shapes (MetaFrames *frames,
         {
           const int width = floor(0.5 + radius - sqrt(radius*radius - (radius-(i+0.5))*(radius-(i+0.5))));
           rect.x = 0;
-          rect.y = new_window_height - i - 1;
+          rect.y = window_height - i - 1;
           rect.width = width;
           rect.height = 1;
 
@@ -943,17 +908,17 @@ meta_frames_apply_shapes (MetaFrames *frames,
         }
     }
 
-  if (fgeom.bottom_right_corner_rounded_radius != 0)
+  if (fgeom->bottom_right_corner_rounded_radius != 0)
     {
-      const int corner = fgeom.bottom_right_corner_rounded_radius * scale;
+      const int corner = fgeom->bottom_right_corner_rounded_radius * scale;
       const float radius = sqrt(corner) + corner;
       int i;
 
       for (i=0; i<corner; i++)
         {
           const int width = floor(0.5 + radius - sqrt(radius*radius - (radius-(i+0.5))*(radius-(i+0.5))));
-          rect.x = new_window_width - width;
-          rect.y = new_window_height - i - 1;
+          rect.x = window_width - width;
+          rect.y = window_height - i - 1;
           rect.width = width;
           rect.height = 1;
 
@@ -961,19 +926,89 @@ meta_frames_apply_shapes (MetaFrames *frames,
         }
     }
 
-  window_region = cairo_region_create ();
+  bounds_region = cairo_region_create ();
 
   rect.x = 0;
   rect.y = 0;
-  rect.width = new_window_width;
-  rect.height = new_window_height;
+  rect.width = window_width;
+  rect.height = window_height;
 
-  cairo_region_union_rectangle (window_region, &rect);
+  cairo_region_union_rectangle (bounds_region, &rect);
 
-  cairo_region_subtract (window_region, corners_region);
+  cairo_region_subtract (bounds_region, corners_region);
 
   cairo_region_destroy (corners_region);
 
+  return bounds_region;
+}
+
+static cairo_region_t *
+get_client_region (MetaFrameGeometry *fgeom,
+                   int                window_width,
+                   int                window_height)
+{
+  cairo_rectangle_int_t rect;
+
+  rect.x = fgeom->left_width;
+  rect.y = fgeom->top_height;
+  rect.width = window_width - fgeom->right_width - rect.x;
+  rect.height = window_height - fgeom->bottom_height - rect.y;
+
+  return cairo_region_create_rectangle (&rect);
+}
+
+void
+meta_frames_apply_shapes (MetaFrames *frames,
+                          Window      xwindow,
+                          int         new_window_width,
+                          int         new_window_height,
+                          gboolean    window_has_shape)
+{
+#ifdef HAVE_SHAPE
+  /* Apply shapes as if window had new_window_width, new_window_height */
+  MetaUIFrame *frame;
+  MetaFrameGeometry fgeom;
+  cairo_region_t *window_region;
+  Display *display;
+
+  frame = meta_frames_lookup_window (frames, xwindow);
+  g_return_if_fail (frame != NULL);
+
+  display = GDK_DISPLAY_XDISPLAY (gdk_display_get_default ());
+
+  meta_frames_calc_geometry (frames, frame, &fgeom);
+
+  if (!(fgeom.top_left_corner_rounded_radius != 0 ||
+        fgeom.top_right_corner_rounded_radius != 0 ||
+        fgeom.bottom_left_corner_rounded_radius != 0 ||
+        fgeom.bottom_right_corner_rounded_radius != 0 ||
+        window_has_shape))
+    {
+      if (frame->shape_applied)
+        {
+          meta_topic (META_DEBUG_SHAPES,
+                      "Unsetting shape mask on frame 0x%lx\n",
+                      frame->xwindow);
+
+          XShapeCombineMask (display, frame->xwindow,
+                             ShapeBounding, 0, 0, None, ShapeSet);
+          frame->shape_applied = FALSE;
+        }
+      else
+        {
+          meta_topic (META_DEBUG_SHAPES,
+                      "Frame 0x%lx still doesn't need a shape mask\n",
+                      frame->xwindow);
+        }
+
+      return; /* nothing to do */
+    }
+
+  window_region = get_bounds_region (frames,
+                                     frame,
+                                     &fgeom,
+                                     new_window_width,
+                                     new_window_height);
   if (window_has_shape)
     {
       /* The client window is oclock or something and has a shape
@@ -1025,14 +1060,9 @@ meta_frames_apply_shapes (MetaFrames *frames,
       /* Punch the client area out of the normal frame shape,
        * then union it with the shape_window's existing shape
        */
-      client_region = cairo_region_create ();
-
-      rect.x = fgeom.left_width;
-      rect.y = fgeom.top_height;
-      rect.width = new_window_width - fgeom.right_width - rect.x;
-      rect.height = new_window_height - fgeom.bottom_height - rect.y;
-
-      cairo_region_union_rectangle (client_region, &rect);
+      client_region = get_client_region (&fgeom,
+                                         new_window_width,
+                                         new_window_height);
 
       cairo_region_subtract (window_region, client_region);
 
@@ -1068,6 +1098,27 @@ meta_frames_apply_shapes (MetaFrames *frames,
 #endif /* HAVE_SHAPE */
 }
 
+cairo_region_t *
+meta_frames_get_frame_bounds (MetaFrames *frames,
+                              Window      xwindow,
+                              int         window_width,
+                              int         window_height)
+{
+  MetaUIFrame *frame;
+  MetaFrameGeometry fgeom;
+
+  frame = meta_frames_lookup_window (frames, xwindow);
+  g_return_val_if_fail (frame != NULL, NULL);
+
+  meta_frames_calc_geometry (frames, frame, &fgeom);
+
+  return get_bounds_region (frames,
+                            frame,
+                            &fgeom,
+                            window_width,
+                            window_height);
+}
+
 void
 meta_frames_move_resize_frame (MetaFrames *frames,
                                Window      xwindow,
diff --git a/src/ui/frames.h b/src/ui/frames.h
index b27f3b11..0af5ed89 100644
--- a/src/ui/frames.h
+++ b/src/ui/frames.h
@@ -140,6 +140,10 @@ void meta_frames_apply_shapes (MetaFrames *frames,
                                int         new_window_width,
                                int         new_window_height,
                                gboolean    window_has_shape);
+cairo_region_t *meta_frames_get_frame_bounds (MetaFrames *frames,
+                                              Window      xwindow,
+                                              int         window_width,
+                                              int         window_height);
 void meta_frames_move_resize_frame (MetaFrames *frames,
 				    Window      xwindow,
 				    int         x,
diff --git a/src/ui/ui.c b/src/ui/ui.c
index 7abbdeaa..070bf127 100644
--- a/src/ui/ui.c
+++ b/src/ui/ui.c
@@ -452,6 +452,18 @@ meta_ui_apply_frame_shape  (MetaUI  *ui,
                             window_has_shape);
 }
 
+cairo_region_t *
+meta_ui_get_frame_bounds (MetaUI *ui,
+                          Window  xwindow,
+                          int     window_width,
+                          int     window_height)
+{
+  return meta_frames_get_frame_bounds (ui->frames,
+                                       xwindow,
+                                       window_width,
+                                       window_height);
+}
+
 void
 meta_ui_queue_frame_draw (MetaUI *ui,
                           Window xwindow)