Get win32 file dialog working again

Signed-off-by: falkTX <falktx@falktx.com>

dgl/Window.hpp

@@ -317,8 +317,10 @@ public:
       Open a file browser dialog with this window as parent.
       A few options can be specified to setup the dialog.
 
-      This function does not block.
       If a path is selected, onFileSelected() will be called with the user chosen path.
+      If the user cancels or does not pick a file, onFileSelected() will be called with nullptr as filename.
+
+      This function does not block the event loop.
     */
     bool openFileBrowser(const FileBrowserOptions& options);
 #endif

dgl/src/WindowPrivateData.cpp

@@ -23,6 +23,9 @@
 
 #ifdef DISTRHO_OS_WINDOWS
 # include <direct.h>
+# include <winsock2.h>
+# include <windows.h>
+# include <vector>
 #else
 # include <unistd.h>
 #endif
@@ -50,6 +53,11 @@ START_NAMESPACE_DGL
 
 // -----------------------------------------------------------------------
 
+#ifdef DISTRHO_OS_WINDOWS
+// static pointer used for direct comparisons
+static const char* const kWin32SelectedFileCancelled = "__dpf_cancelled__";
+#endif
+
 static double getDesktopScaleFactor()
 {
     if (const char* const scale = getenv("DPF_SCALE_FACTOR"))
@@ -195,6 +203,11 @@ Window::PrivateData::~PrivateData()
     appData->idleCallbacks.remove(this);
     appData->windows.remove(self);
 
+#ifdef DISTRHO_OS_WINDOWS
+    if (win32SelectedFile != nullptr && win32SelectedFile != kWin32SelectedFileCancelled)
+        std::free(const_cast<char*>(win32SelectedFile));
+#endif
+
     if (view != nullptr)
         puglFreeView(view);
 }
@@ -366,11 +379,13 @@ void Window::PrivateData::idleCallback()
 {
 #ifndef DGL_FILE_BROWSER_DISABLED
 # ifdef DISTRHO_OS_WINDOWS
-    if (char* const path = win32SelectedFile)
+    if (const char* path = win32SelectedFile)
     {
         win32SelectedFile = nullptr;
+        if (path == kWin32SelectedFileCancelled)
+            path = nullptr;
         self->onFileSelected(path);
-        std::free(path);
+        std::free(const_cast<char*>(path));
     }
 # endif
 # ifdef HAVE_X11
@@ -465,6 +480,54 @@ bool Window::PrivateData::openFileBrowser(const Window::FileBrowserOptions& opti
     // --------------------------------------------------------------------------
     // show
 
+#ifdef DISTRHO_OS_WINDOWS
+    // the old and compatible dialog API
+    OPENFILENAMEW ofn;
+    memset(&ofn, 0, sizeof(ofn));
+    if (win32SelectedFile != nullptr && win32SelectedFile != kWin32SelectedFileCancelled)
+        std::free(const_cast<char*>(win32SelectedFile));
+    win32SelectedFile = nullptr;
+
+    ofn.lStructSize = sizeof(ofn);
+    ofn.hwndOwner = (HWND)puglGetNativeWindow(view);
+
+    // set start directory in UTF-16 encoding
+    std::vector<WCHAR> startDirW;
+    startDirW.resize(startDir.length() + 1);
+    if (MultiByteToWideChar(CP_UTF8, 0, startDir.buffer(), -1, startDirW.data(), startDirW.size()))
+        ofn.lpstrInitialDir = startDirW.data();
+
+    // set title in UTF-16 encoding
+    std::vector<WCHAR> titleW;
+    titleW.resize(title.length() + 1);
+    if (MultiByteToWideChar(CP_UTF8, 0, title.buffer(), -1, titleW.data(), titleW.size()))
+        ofn.lpstrTitle = titleW.data();
+
+    // prepare a buffer to receive the result
+    std::vector<WCHAR> fileNameW(32768); // the Unicode maximum
+    ofn.lpstrFile = fileNameW.data();
+    ofn.nMaxFile = (DWORD)fileNameW.size();
+
+    // TODO synchronous only, can't do better with WinAPI native dialogs.
+    // threading might work, if someone is motivated to risk it.
+    if (GetOpenFileNameW(&ofn))
+    {
+        // back to UTF-8
+        std::vector<char> fileNameA(4 * 32768);
+        if (WideCharToMultiByte(CP_UTF8, 0, fileNameW.data(), -1,
+                                fileNameA.data(), (int)fileNameA.size(),
+                                nullptr, nullptr))
+        {
+            // handle it during the next idle cycle (fake async)
+            win32SelectedFile = strdup(fileNameA.data());
+        }
+    }
+
+    if (win32SelectedFile == nullptr)
+        win32SelectedFile = kWin32SelectedFileCancelled;
+
+    return true;
+#endif
 #ifdef HAVE_X11
     uint flags = 0x0;
     // TODO flags

dgl/src/WindowPrivateData.hpp

@@ -70,7 +70,7 @@ struct Window::PrivateData : IdleCallback {
 
 #ifdef DISTRHO_OS_WINDOWS
     /** Selected file for openFileBrowser on windows, stored for fake async operation. */
-    char* win32SelectedFile;
+    const char* win32SelectedFile;
 #endif
 
     /** Modal window setup. */

tests/FileBrowserDialog.cpp

@@ -116,7 +116,11 @@ protected:
 
                 FileBrowserOptions opts;
                 opts.title = "Look at me";
-                openFileBrowser(opts);
+                if (! openFileBrowser(opts))
+                {
+                    selectedFile = "(Failed to start file browser)";
+                    repaint();
+                }
             }
 
             return true;
@@ -133,6 +137,16 @@ protected:
         buttonBounds = Rectangle<uint>(width - 120, height/2 - 20, 100, 40);
     }
 
+    void onFocus(const bool focus, CrossingMode) override
+    {
+        if (focus)
+            return;
+
+        buttonClick = false;
+        buttonHover = false;
+        repaint();
+    }
+
     void onFileSelected(const char* filename) override
     {
         if (filename == nullptr)