Rework code to find transient windows to map

source/backend/plugin/CarlaPluginJack.cpp

@@ -120,13 +120,9 @@ protected:
             libjackdir += "/jack";
 
             CarlaString ldpreload;
-
 #ifdef HAVE_X11
-            if (options.frontendWinId != 0)
-            {
-                ldpreload = (CarlaString(options.binaryDir)
-                          + "/libcarla_interposer-jack-x11.so");
-            }
+            ldpreload = (CarlaString(options.binaryDir)
+                      + "/libcarla_interposer-jack-x11.so");
 #endif
 
             const ScopedEngineEnvironmentLocker _seel(kEngine);

source/interposer/interposer-jack-x11.cpp

@@ -40,43 +40,69 @@ typedef int (*XMapWindowFunc)(Display*, Window);
 typedef int (*XUnmapWindowFunc)(Display*, Window);
 
 // -----------------------------------------------------------------------
-// Global counter so we only map the first (hopefully main) window
+// Current mapped window
 
-static int sMapWindowCounter = 0;
+static Window sCurrentlyMappedWindow = 0;
 
 // -----------------------------------------------------------------------
 // Calling the real functions
 
-static int real_XMapWindow(Display* display, Window w)
+static int real_XMapWindow(Display* display, Window window)
 {
     static const XMapWindowFunc func = (XMapWindowFunc)::dlsym(RTLD_NEXT, "XMapWindow");
     CARLA_SAFE_ASSERT_RETURN(func != nullptr, 0);
 
-    return func(display, w);
+    return func(display, window);
 }
 
-static int real_XUnmapWindow(Display* display, Window w)
+static int real_XUnmapWindow(Display* display, Window window)
 {
     static const XUnmapWindowFunc func = (XUnmapWindowFunc)::dlsym(RTLD_NEXT, "XUnmapWindow");
     CARLA_SAFE_ASSERT_RETURN(func != nullptr, 0);
 
-    return func(display, w);
+    return func(display, window);
 }
 
 // -----------------------------------------------------------------------
 // Our custom functions
 
 CARLA_EXPORT
-int XMapWindow(Display* display, Window w)
+int XMapWindow(Display* display, Window window)
 {
-    carla_stdout("------------------------------- XMapWindow called");
+    static const ScopedLibOpen slo;
 
     for (;;)
     {
-        if (++sMapWindowCounter != 1)
+        if (sCurrentlyMappedWindow != 0)
             break;
 
-        static const ScopedLibOpen slo;
+        Atom atom;
+        int atomFormat;
+        unsigned char* atomPtrs;
+        unsigned long numItems, ignored;
+
+        const Atom wmWindowType = XInternAtom(display, "_NET_WM_WINDOW_TYPE", True);
+
+        if (XGetWindowProperty(display, window, wmWindowType, 0, ~0L, False, AnyPropertyType,
+                               &atom, &atomFormat, &numItems, &ignored, &atomPtrs) == Success)
+        {
+            const Atom* const atomValues = (const Atom*)atomPtrs;
+
+            for (ulong i=0; i<numItems; ++i)
+            {
+                const char* const atomValue(XGetAtomName(display, atomValues[i]));
+                CARLA_SAFE_ASSERT_CONTINUE(atomValue != nullptr && atomValue[0] != '\0');
+
+                if (std::strcmp(atomValue, "_NET_WM_WINDOW_TYPE_NORMAL") == 0)
+                {
+                    sCurrentlyMappedWindow = window;
+                    break;
+                }
+            }
+        }
+
+        if (sCurrentlyMappedWindow == 0)
+            break;
 
         if (const char* const winIdStr = std::getenv("CARLA_FRONTEND_WIN_ID"))
         {
@@ -86,7 +112,7 @@ int XMapWindow(Display* display, Window w)
             CARLA_SAFE_ASSERT_BREAK(winIdLL > 0);
 
             const Window winId(static_cast<Window>(winIdLL));
-            XSetTransientForHint(display, w, static_cast<Window>(winId));
+            XSetTransientForHint(display, window, static_cast<Window>(winId));
 
             carla_stdout("Transient hint correctly applied before mapping window");
         }
@@ -94,17 +120,16 @@ int XMapWindow(Display* display, Window w)
         break;
     }
 
-    return real_XMapWindow(display, w);
+    return real_XMapWindow(display, window);
 }
 
 CARLA_EXPORT
-int XUnmapWindow(Display* display, Window w)
+int XUnmapWindow(Display* display, Window window)
 {
-    carla_stdout("------------------------------- XUnmapWindow called");
-
-    --sMapWindowCounter;
+    if (sCurrentlyMappedWindow == window)
+        sCurrentlyMappedWindow = 0;
 
-    return real_XUnmapWindow(display, w);
+    return real_XUnmapWindow(display, window);
 }
 
 // -----------------------------------------------------------------------

source/interposer/interposer-x11.cpp

@@ -27,42 +27,70 @@ typedef int (*XMapWindowFunc)(Display*, Window);
 typedef int (*XUnmapWindowFunc)(Display*, Window);
 
 // -----------------------------------------------------------------------
-// Global counter so we only map the first (hopefully main) window
+// Current mapped window
 
-static int sMapWindowCounter = 0;
+static Window sCurrentlyMappedWindow = 0;
 
 // -----------------------------------------------------------------------
 // Calling the real functions
 
-static int real_XMapWindow(Display* display, Window w)
+static int real_XMapWindow(Display* display, Window window)
 {
     static const XMapWindowFunc func = (XMapWindowFunc)::dlsym(RTLD_NEXT, "XMapWindow");
     CARLA_SAFE_ASSERT_RETURN(func != nullptr, 0);
 
-    return func(display, w);
+    return func(display, window);
 }
 
-static int real_XUnmapWindow(Display* display, Window w)
+static int real_XUnmapWindow(Display* display, Window window)
 {
     static const XUnmapWindowFunc func = (XUnmapWindowFunc)::dlsym(RTLD_NEXT, "XUnmapWindow");
     CARLA_SAFE_ASSERT_RETURN(func != nullptr, 0);
 
-    return func(display, w);
+    return func(display, window);
 }
 
 // -----------------------------------------------------------------------
 // Our custom functions
 
 CARLA_EXPORT
-int XMapWindow(Display* display, Window w)
+int XMapWindow(Display* display, Window window)
 {
-    carla_stdout("------------------------------- XMapWindow called");
-
     for (;;)
     {
-        if (++sMapWindowCounter != 1)
+#if 0
+        if (sCurrentlyMappedWindow != 0)
             break;
 
+        Atom atom;
+        int atomFormat;
+        unsigned char* atomPtrs;
+        unsigned long numItems, ignored;
+
+        const Atom wmWindowType = XInternAtom(display, "_NET_WM_WINDOW_TYPE", True);
+
+        if (XGetWindowProperty(display, window, wmWindowType, 0, ~0L, False, AnyPropertyType,
+                               &atom, &atomFormat, &numItems, &ignored, &atomPtrs) == Success)
+        {
+            const Atom* const atomValues = (const Atom*)atomPtrs;
+
+            for (ulong i=0; i<numItems; ++i)
+            {
+                const char* const atomValue(XGetAtomName(display, atomValues[i]));
+                CARLA_SAFE_ASSERT_CONTINUE(atomValue != nullptr && atomValue[0] != '\0');
+
+                if (std::strcmp(atomValue, "_NET_WM_WINDOW_TYPE_NORMAL") == 0)
+                {
+                    sCurrentlyMappedWindow = window;
+                    break;
+                }
+            }
+        }
+#endif
+
+        if (sCurrentlyMappedWindow == 0)
+            sCurrentlyMappedWindow = window;
+
         if (const char* const winIdStr = std::getenv("CARLA_ENGINE_OPTION_FRONTEND_WIN_ID"))
         {
             CARLA_SAFE_ASSERT_BREAK(winIdStr[0] != '\0');
@@ -71,7 +99,7 @@ int XMapWindow(Display* display, Window w)
             CARLA_SAFE_ASSERT_BREAK(winIdLL > 0);
 
             const Window winId(static_cast<Window>(winIdLL));
-            XSetTransientForHint(display, w, static_cast<Window>(winId));
+            XSetTransientForHint(display, window, static_cast<Window>(winId));
 
             carla_stdout("Transient hint correctly applied before mapping window");
         }
@@ -79,17 +107,16 @@ int XMapWindow(Display* display, Window w)
         break;
     }
 
-    return real_XMapWindow(display, w);
+    return real_XMapWindow(display, window);
 }
 
 CARLA_EXPORT
-int XUnmapWindow(Display* display, Window w)
+int XUnmapWindow(Display* display, Window window)
 {
-    carla_stdout("------------------------------- XUnmapWindow called");
-
-    --sMapWindowCounter;
+    if (sCurrentlyMappedWindow == window)
+        sCurrentlyMappedWindow = 0;
 
-    return real_XUnmapWindow(display, w);
+    return real_XUnmapWindow(display, window);
 }
 
 // -----------------------------------------------------------------------

source/utils/CarlaPluginUI.cpp

@@ -621,7 +621,7 @@ bool CarlaPluginUI::tryTransientWinIdMatch(const uintptr_t pid, const char* cons
     CARLA_SAFE_ASSERT_RETURN(numWindows != 0, true);
 
     Window* windows = (Window*)data;
-    Window  lastGoodWindow = 0;
+    Window  lastGoodWindowPID = 0, lastGoodWindowNameSimple = 0, lastGoodWindowNameUTF8 = 0;
 
     for (ulong i = 0; i < numWindows; i++)
     {
@@ -646,12 +646,7 @@ bool CarlaPluginUI::tryTransientWinIdMatch(const uintptr_t pid, const char* cons
                 CARLA_SAFE_ASSERT_CONTINUE(pidSize != 0);
 
                 if (*(ulong*)pidData == static_cast<ulong>(pid))
-                {
-                    CARLA_SAFE_ASSERT_RETURN(lastGoodWindow == window || lastGoodWindow == 0, true);
-                    lastGoodWindow = window;
-                    carla_stdout("Match found using pid");
-                    break;
-                }
+                    lastGoodWindowPID = window;
             }
         }
 
@@ -675,9 +670,10 @@ bool CarlaPluginUI::tryTransientWinIdMatch(const uintptr_t pid, const char* cons
 
             if (std::strstr((const char*)nameData, uiTitle) != nullptr)
             {
-                CARLA_SAFE_ASSERT_RETURN(lastGoodWindow == window || lastGoodWindow == 0,  true);
-                lastGoodWindow = window;
-                carla_stdout("Match found using UTF-8 name");
+                lastGoodWindowNameUTF8 = window;
+
+                if (lastGoodWindowPID == window)
+                    break;
             }
         }
 
@@ -696,29 +692,74 @@ bool CarlaPluginUI::tryTransientWinIdMatch(const uintptr_t pid, const char* cons
 
             if (std::strstr(wmName, uiTitle) != nullptr)
             {
-                CARLA_SAFE_ASSERT_RETURN(lastGoodWindow == window || lastGoodWindow == 0,  true);
-                lastGoodWindow = window;
-                carla_stdout("Match found using simple name");
+                lastGoodWindowNameSimple = window;
+
+                if (lastGoodWindowPID == window)
+                    break;
             }
         }
     }
 
-    if (lastGoodWindow == 0)
+    if (lastGoodWindowPID == 0 && lastGoodWindowNameSimple == 0 && lastGoodWindowNameUTF8 == 0)
         return false;
 
+    Window windowToMap;
+
+    if (lastGoodWindowPID != 0)
+    {
+        if (lastGoodWindowPID == lastGoodWindowNameSimple && lastGoodWindowPID == lastGoodWindowNameUTF8)
+        {
+            carla_stdout("Match found using pid, simple and UTF-8 name all at once, nice!");
+            windowToMap = lastGoodWindowPID;
+        }
+        else if (lastGoodWindowPID == lastGoodWindowNameUTF8)
+        {
+            carla_stdout("Match found using pid and UTF-8 name");
+            windowToMap = lastGoodWindowPID;
+        }
+        else if (lastGoodWindowPID == lastGoodWindowNameSimple)
+        {
+            carla_stdout("Match found using pid and simple name");
+            windowToMap = lastGoodWindowPID;
+        }
+        else
+        {
+            carla_stdout("Match found using pid");
+            windowToMap = lastGoodWindowPID;
+        }
+    }
+    else if (lastGoodWindowNameUTF8 != 0)
+    {
+        if (lastGoodWindowNameUTF8 == lastGoodWindowNameSimple)
+        {
+            carla_stdout("Match found using simple and UTF-8 name");
+            windowToMap = lastGoodWindowNameUTF8;
+        }
+        else
+        {
+            carla_stdout("Match found using simple and UTF-8 name");
+            windowToMap = lastGoodWindowNameUTF8;
+        }
+    }
+    else
+    {
+        carla_stdout("Match found using simple name");
+        windowToMap = lastGoodWindowNameSimple;
+    }
+
     const Atom _nwt    = XInternAtom(sd.display ,"_NET_WM_STATE", False);
     const Atom _nws[2] = {
         XInternAtom(sd.display, "_NET_WM_STATE_SKIP_TASKBAR", False),
         XInternAtom(sd.display, "_NET_WM_STATE_SKIP_PAGER", False)
     };
-    XChangeProperty(sd.display, lastGoodWindow, _nwt, XA_ATOM, 32, PropModeAppend, (const uchar*)_nws, 2);
+    XChangeProperty(sd.display, windowToMap, _nwt, XA_ATOM, 32, PropModeAppend, (const uchar*)_nws, 2);
 
     const Atom _nwi = XInternAtom(sd.display, "_NET_WM_ICON", False);
-    XChangeProperty(sd.display, lastGoodWindow, _nwi, XA_CARDINAL, 32, PropModeReplace, (const uchar*)sCarlaX11Icon, sCarlaX11IconSize);
+    XChangeProperty(sd.display, windowToMap, _nwi, XA_CARDINAL, 32, PropModeReplace, (const uchar*)sCarlaX11Icon, sCarlaX11IconSize);
 
     const Window hostWinId((Window)winId);
 
-    XSetTransientForHint(sd.display, lastGoodWindow, hostWinId);
+    XSetTransientForHint(sd.display, windowToMap, hostWinId);
 
     if (centerUI && false /* moving the window after being shown isn't pretty... */)
     {
@@ -727,15 +768,15 @@ bool CarlaPluginUI::tryTransientWinIdMatch(const uintptr_t pid, const char* cons
         Window retWindow;
 
         if (XGetGeometry(sd.display, hostWinId,      &retWindow, &hostX,   &hostY,   &hostWidth,   &hostHeight,   &border, &depth) != 0 &&
-            XGetGeometry(sd.display, lastGoodWindow, &retWindow, &pluginX, &pluginY, &pluginWidth, &pluginHeight, &border, &depth) != 0)
+            XGetGeometry(sd.display, windowToMap, &retWindow, &pluginX, &pluginY, &pluginWidth, &pluginHeight, &border, &depth) != 0)
         {
             if (XTranslateCoordinates(sd.display, hostWinId,      rootWindow, hostX,   hostY,   &hostX,   &hostY,   &retWindow) == True &&
-                XTranslateCoordinates(sd.display, lastGoodWindow, rootWindow, pluginX, pluginY, &pluginX, &pluginY, &retWindow) == True)
+                XTranslateCoordinates(sd.display, windowToMap, rootWindow, pluginX, pluginY, &pluginX, &pluginY, &retWindow) == True)
             {
                 const int newX = hostX + int(hostWidth/2  - pluginWidth/2);
                 const int newY = hostY + int(hostHeight/2 - pluginHeight/2);
 
-                XMoveWindow(sd.display, lastGoodWindow, newX, newY);
+                XMoveWindow(sd.display, windowToMap, newX, newY);
             }
         }
     }
@@ -744,8 +785,8 @@ bool CarlaPluginUI::tryTransientWinIdMatch(const uintptr_t pid, const char* cons
     XRaiseWindow(sd.display, hostWinId);
     XSetInputFocus(sd.display, hostWinId, RevertToPointerRoot, CurrentTime);
 
-    XRaiseWindow(sd.display, lastGoodWindow);
-    XSetInputFocus(sd.display, lastGoodWindow, RevertToPointerRoot, CurrentTime);
+    XRaiseWindow(sd.display, windowToMap);
+    XSetInputFocus(sd.display, windowToMap, RevertToPointerRoot, CurrentTime);
 
     XFlush(sd.display);
     return true;