Only shadow ARGB windows with a frame outside the frame

An ARGB window with a frame is likely something like a transparent
terminal. It looks awful (and breaks transparency) to draw a big
opaque black shadow under the window, so clip out the region under
the terminal from the shadow we draw.

Add meta_window_get_frame_bounds() to get a cairo region for the
outer bounds of the frame of a window, and modify the frame handling
code to notice changes to the frame shape and discard a cached
region. meta_frames_apply_shapes() is refactored so we can extract
meta_frames_get_frame_bounds() from it.

https://bugzilla.gnome.org/show_bug.cgi?id=635268

NOTE: Applied only partially, compositor part is still missing...

upstream commit:
https://gitlab.gnome.org/GNOME/metacity/commit/0f2e32d1

4 files changed, 61 insertions, 11 deletions

diff --git a/src/core/frame-private.h b/src/core/frame-private.h
index 1b995f58..d7183149 100644
--- a/src/core/frame-private.h
+++ b/src/core/frame-private.h
@@ -59,10 +59,11 @@ void     meta_frame_queue_draw              (MetaFrame  *frame);
 
 MetaFrameFlags meta_frame_get_flags (MetaFrame *frame);
 
-void meta_frame_sync_to_window    (MetaFrame        *frame,
+gboolean meta_frame_sync_to_window (MetaFrame         *frame,
                                    int               gravity,
                                    gboolean          need_move,
                                    gboolean          need_resize);
+cairo_region_t *meta_frame_get_frame_bounds (MetaFrame *frame);
 
 void meta_frame_set_screen_cursor (MetaFrame        *frame,
                                    MetaCursor        cursor);
diff --git a/src/core/frame.c b/src/core/frame.c
index 25f83ceb..d2f5cfd3 100644
--- a/src/core/frame.c
+++ b/src/core/frame.c
@@ -316,7 +316,7 @@ meta_frame_calc_borders (MetaFrame        *frame,
                              borders);
 }
 
-static void
+static gboolean
 update_shape (MetaFrame *frame)
 {
   if (frame->need_reapply_frame_shape)
@@ -327,10 +327,13 @@ update_shape (MetaFrame *frame)
                                  frame->rect.height,
                                  frame->window->has_shape);
       frame->need_reapply_frame_shape = FALSE;
+      return TRUE;
     }
+  else
+    return FALSE;
 }
 
-void
+gboolean
 meta_frame_sync_to_window (MetaFrame *frame,
                            int        resize_gravity,
                            gboolean   need_move,
@@ -338,8 +341,7 @@ meta_frame_sync_to_window (MetaFrame *frame,
 {
   if (!(need_move || need_resize))
     {
-      update_shape (frame);
-      return;
+      return update_shape (frame);
     }
 
   meta_topic (META_DEBUG_GEOMETRY,
@@ -381,6 +383,16 @@ meta_frame_sync_to_window (MetaFrame *frame,
         meta_ui_repaint_frame (frame->window->screen->ui,
                                frame->xwindow);
     }
+  return need_resize;
+}
+
+cairo_region_t *
+meta_frame_get_frame_bounds (MetaFrame *frame)
+{
+  return meta_ui_get_frame_bounds (frame->window->screen->ui,
+                                   frame->xwindow,
+                                   frame->rect.width,
+                                   frame->rect.height);
 }
 
 void
diff --git a/src/core/window-private.h b/src/core/window-private.h
index 38184231..50bc57be 100644
--- a/src/core/window-private.h
+++ b/src/core/window-private.h
@@ -41,6 +41,7 @@
 #include "stack.h"
 #include "iconcache.h"
 #include <X11/Xutil.h>
+#include <cairo.h>
 #include <gdk-pixbuf/gdk-pixbuf.h>
 #include <gtk/gtk.h>
 
@@ -324,6 +325,9 @@ struct _MetaWindow
   /* if TRUE, application is buggy and SYNC resizing is turned off */
   guint disable_sync : 1;
 
+  /* if non-NULL, the bounds of the window frame */
+  cairo_region_t *frame_bounds;
+
   /* Note: can be NULL */
   GSList *struts;
 
diff --git a/src/core/window.c b/src/core/window.c
index fded7b8e..d6566de3 100644
--- a/src/core/window.c
+++ b/src/core/window.c
@@ -524,6 +524,7 @@ meta_window_new_with_attrs (MetaDisplay       *display,
   window->shaken_loose = FALSE;
   window->have_focus_click_grab = FALSE;
   window->disable_sync = FALSE;
+  window->frame_bounds = NULL;
 
   window->unmaps_pending = 0;
 
@@ -1221,6 +1222,9 @@ meta_window_free (MetaWindow  *window,
   if (window->mini_icon)
     g_object_unref (G_OBJECT (window->mini_icon));
 
+  if (window->frame_bounds)
+    cairo_region_destroy (window->frame_bounds);
+
   meta_icon_cache_free (&window->icon_cache);
 
   g_free (window->sm_client_id);
@@ -3483,6 +3487,7 @@ meta_window_move_resize_internal (MetaWindow          *window,
   int frame_size_dy;
   int size_dx;
   int size_dy;
+  gboolean frame_shape_changed = FALSE;
   gboolean is_configure_request;
   gboolean do_gravity_adjust;
   gboolean is_user_action;
@@ -3789,9 +3794,10 @@ meta_window_move_resize_internal (MetaWindow          *window,
     meta_window_set_gravity (window, StaticGravity);
 
   if (configure_frame_first && have_window_frame)
-    meta_frame_sync_to_window (window->frame,
-                               gravity,
-                               need_move_frame, need_resize_frame);
+    frame_shape_changed = meta_frame_sync_to_window (window->frame,
+                                                     gravity,
+                                                     need_move_frame,
+                                                     need_resize_frame);
 
   values.border_width = 0;
   values.x = client_move_x;
@@ -3846,9 +3852,10 @@ meta_window_move_resize_internal (MetaWindow          *window,
     }
 
   if (!configure_frame_first && have_window_frame)
-    meta_frame_sync_to_window (window->frame,
-                               gravity,
-                               need_move_frame, need_resize_frame);
+    frame_shape_changed = meta_frame_sync_to_window (window->frame,
+                                                     gravity,
+                                                     need_move_frame,
+                                                     need_resize_frame);
 
   /* Put gravity back to be nice to lesser window managers */
   if (use_static_gravity)
@@ -3888,6 +3895,11 @@ meta_window_move_resize_internal (MetaWindow          *window,
    *      server-side size/pos of window->xwindow and frame->xwindow
    *   b) all constraints are obeyed by window->rect and frame->rect
    */
+  if (frame_shape_changed && window->frame_bounds)
+    {
+      cairo_region_destroy (window->frame_bounds);
+      window->frame_bounds = NULL;
+    }
 
   if (meta_prefs_get_attach_modal_dialogs ())
     meta_window_foreach_transient (window, move_attached_dialog, NULL);
@@ -8764,3 +8776,24 @@ meta_window_is_client_decorated (MetaWindow *window)
    */
   return window->has_custom_frame_extents;
 }
+
+/**
+ * meta_window_get_frame_bounds:
+ *
+ * Gets a region representing the outer bounds of the window's frame.
+ *
+ * Return value: (transfer none) (allow-none): a #cairo_region_t
+ * holding the outer bounds of the window, or %NULL if the window
+ * doesn't have a frame.
+ */
+cairo_region_t *
+meta_window_get_frame_bounds (MetaWindow *window)
+{
+  if (!window->frame_bounds)
+    {
+      if (window->frame)
+        window->frame_bounds = meta_frame_get_frame_bounds (window->frame);
+    }
+
+  return window->frame_bounds;
+}