Plugin scanning: ability to use multiple threads

modules/juce_audio_processors/scanning/juce_KnownPluginList.cpp

@@ -126,6 +126,8 @@ bool KnownPluginList::scanAndAddFile (const String& fileOrIdentifier,
                                       OwnedArray <PluginDescription>& typesFound,
                                       AudioPluginFormat& format)
 {
+    const ScopedLock sl (scanLock);
+
     if (dontRescanIfAlreadyInList
          && getTypeForFile (fileOrIdentifier) != nullptr)
     {
@@ -153,14 +155,17 @@ bool KnownPluginList::scanAndAddFile (const String& fileOrIdentifier,
 
     OwnedArray <PluginDescription> found;
 
-    if (scanner != nullptr)
     {
-        if (! scanner->findPluginTypesFor (format, found, fileOrIdentifier))
-            addToBlacklist (fileOrIdentifier);
-    }
-    else
-    {
-        format.findAllTypesForFile (found, fileOrIdentifier);
+        const ScopedUnlock sl (scanLock);
+        if (scanner != nullptr)
+        {
+            if (! scanner->findPluginTypesFor (format, found, fileOrIdentifier))
+                addToBlacklist (fileOrIdentifier);
+        }
+        else
+        {
+            format.findAllTypesForFile (found, fileOrIdentifier);
+        }
     }
 
     for (int i = 0; i < found.size(); ++i)

modules/juce_audio_processors/scanning/juce_KnownPluginList.h

@@ -201,6 +201,7 @@ private:
     OwnedArray <PluginDescription> types;
     StringArray blacklist;
     ScopedPointer<CustomScanner> scanner;
+    CriticalSection scanLock;
 
     JUCE_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (KnownPluginList)
 };

modules/juce_audio_processors/scanning/juce_PluginDirectoryScanner.cpp

@@ -39,7 +39,6 @@ PluginDirectoryScanner::PluginDirectoryScanner (KnownPluginList& listToAddTo,
     : list (listToAddTo),
       format (formatToLookFor),
       deadMansPedalFile (deadMansPedal),
-      nextIndex (0),
       progress (0)
 {
     directoriesToSearch.removeRedundantPaths();
@@ -60,6 +59,7 @@ PluginDirectoryScanner::PluginDirectoryScanner (KnownPluginList& listToAddTo,
     }
 
     applyBlacklistingsFromDeadMansPedal (listToAddTo, deadMansPedalFile);
+    nextIndex.set (filesOrIdentifiersToScan.size());
 }
 
 PluginDirectoryScanner::~PluginDirectoryScanner()
@@ -69,43 +69,51 @@ PluginDirectoryScanner::~PluginDirectoryScanner()
 //==============================================================================
 String PluginDirectoryScanner::getNextPluginFileThatWillBeScanned() const
 {
-    return format.getNameOfPluginFromIdentifier (filesOrIdentifiersToScan [nextIndex]);
+    return format.getNameOfPluginFromIdentifier (filesOrIdentifiersToScan [nextIndex.get() - 1]);
+}
+
+void PluginDirectoryScanner::updateProgress()
+{
+    progress = (1.0f - nextIndex.get() / (float) filesOrIdentifiersToScan.size());
 }
 
 bool PluginDirectoryScanner::scanNextFile (const bool dontRescanIfAlreadyInList)
 {
-    String file (filesOrIdentifiersToScan [nextIndex]);
+    const int index = --nextIndex;
 
-    if (file.isNotEmpty() && ! list.isListingUpToDate (file))
+    if (index >= 0)
     {
-        OwnedArray <PluginDescription> typesFound;
+        String file (filesOrIdentifiersToScan [index]);
 
-        // Add this plugin to the end of the dead-man's pedal list in case it crashes...
-        StringArray crashedPlugins (readDeadMansPedalFile (deadMansPedalFile));
-        crashedPlugins.removeString (file);
-        crashedPlugins.add (file);
-        setDeadMansPedalFile (crashedPlugins);
+        if (file.isNotEmpty() && ! list.isListingUpToDate (file))
+        {
+            OwnedArray <PluginDescription> typesFound;
 
-        list.scanAndAddFile (file, dontRescanIfAlreadyInList, typesFound, format);
+            // Add this plugin to the end of the dead-man's pedal list in case it crashes...
+            StringArray crashedPlugins (readDeadMansPedalFile (deadMansPedalFile));
+            crashedPlugins.removeString (file);
+            crashedPlugins.add (file);
+            setDeadMansPedalFile (crashedPlugins);
 
-        // Managed to load without crashing, so remove it from the dead-man's-pedal..
-        crashedPlugins.removeString (file);
-        setDeadMansPedalFile (crashedPlugins);
+            list.scanAndAddFile (file, dontRescanIfAlreadyInList, typesFound, format);
 
-        if (typesFound.size() == 0 && ! list.getBlacklistedFiles().contains (file))
-            failedFiles.add (file);
+            // Managed to load without crashing, so remove it from the dead-man's-pedal..
+            crashedPlugins.removeString (file);
+            setDeadMansPedalFile (crashedPlugins);
+
+            if (typesFound.size() == 0 && ! list.getBlacklistedFiles().contains (file))
+                failedFiles.add (file);
+        }
     }
 
-    return skipNextFile();
+    updateProgress();
+    return index > 0;
 }
 
 bool PluginDirectoryScanner::skipNextFile()
 {
-    if (nextIndex >= filesOrIdentifiersToScan.size())
-        return false;
-
-    progress = ++nextIndex / (float) filesOrIdentifiersToScan.size();
-    return nextIndex < filesOrIdentifiersToScan.size();
+    updateProgress();
+    return --nextIndex > 0;
 }
 
 void PluginDirectoryScanner::setDeadMansPedalFile (const StringArray& newContents)

modules/juce_audio_processors/scanning/juce_PluginDirectoryScanner.h

@@ -115,9 +115,10 @@ private:
     StringArray filesOrIdentifiersToScan;
     File deadMansPedalFile;
     StringArray failedFiles;
-    int nextIndex;
+    Atomic<int> nextIndex;
     float progress;
 
+    void updateProgress();
     void setDeadMansPedalFile (const StringArray& newContents);
 
     JUCE_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (PluginDirectoryScanner)

modules/juce_audio_processors/scanning/juce_PluginListComponent.cpp

@@ -32,7 +32,7 @@ PluginListComponent::PluginListComponent (AudioPluginFormatManager& manager,
       deadMansPedalFile (deadMansPedal),
       optionsButton ("Options..."),
       propertiesToUse (properties),
-      scanOnBackgroundThread (false)
+      numThreads (0)
 {
     listBox.setModel (this);
     addAndMakeVisible (&listBox);
@@ -60,9 +60,9 @@ void PluginListComponent::setOptionsButtonText (const String& newText)
     resized();
 }
 
-void PluginListComponent::setScansOnMessageThread (bool useMessageThread) noexcept
+void PluginListComponent::setNumberOfThreadsForScanning (int num)
 {
-    scanOnBackgroundThread = ! useMessageThread;
+    numThreads = num;
 }
 
 void PluginListComponent::resized()
@@ -244,20 +244,18 @@ void PluginListComponent::filesDropped (const StringArray& files, int, int)
 }
 
 //==============================================================================
-class PluginListComponent::Scanner    : private Timer,
-                                        private Thread
+class PluginListComponent::Scanner    : private Timer
 {
 public:
     Scanner (PluginListComponent& plc,
              AudioPluginFormat& format,
              PropertiesFile* properties,
-             bool useThread)
-        : Thread ("plugin_scan"),
-          owner (plc), formatToScan (format), propertiesToUse (properties),
+             int threads)
+        : owner (plc), formatToScan (format), propertiesToUse (properties),
           pathChooserWindow (TRANS("Select folders to scan..."), String::empty, AlertWindow::NoIcon),
           progressWindow (TRANS("Scanning for plug-ins..."),
                           TRANS("Searching for all possible plug-in files..."), AlertWindow::NoIcon),
-          progress (0.0), shouldUseThread (useThread), finished (false)
+          progress (0.0), numThreads (threads), finished (false)
     {
         FileSearchPath path (formatToScan.getDefaultLocationsToSearch());
 
@@ -285,7 +283,11 @@ public:
 
     ~Scanner()
     {
-        stopThread (10000);
+        if (pool != nullptr)
+        {
+            pool->removeAllJobs (true, 60000);
+            pool = nullptr;
+        }
     }
 
 private:
@@ -317,8 +319,13 @@ private:
         progressWindow.addProgressBarComponent (progress);
         progressWindow.enterModalState();
 
-        if (shouldUseThread)
-            startThread();
+        if (numThreads > 0)
+        {
+            pool = new ThreadPool (numThreads);
+
+            for (int i = numThreads; --i >= 0;)
+                pool->addJob (new ScanJob (*this), true);
+        }
 
         startTimer (20);
     }
@@ -331,7 +338,7 @@ private:
 
     void timerCallback()
     {
-        if (! isThreadRunning())
+        if (pool == nullptr)
         {
             if (doNextScan())
                 startTimer (20);
@@ -346,12 +353,6 @@ private:
             progressWindow.setMessage (progressMessage);
     }
 
-    void run()
-    {
-        while (doNextScan() && ! threadShouldExit())
-        {}
-    }
-
     bool doNextScan()
     {
         progressMessage = TRANS("Testing:\n\n") + scanner->getNextPluginFileThatWillBeScanned();
@@ -374,15 +375,36 @@ private:
     FileSearchPathListComponent pathList;
     String progressMessage;
     double progress;
-    bool shouldUseThread, finished;
+    int numThreads;
+    bool finished;
+
+    ScopedPointer<ThreadPool> pool;
+
+    struct ScanJob  : public ThreadPoolJob
+    {
+        ScanJob (Scanner& s)  : ThreadPoolJob ("pluginscan"), scanner (s) {}
+
+        JobStatus runJob()
+        {
+            while (scanner.doNextScan() && ! shouldExit())
+            {}
+
+            return jobHasFinished;
+        }
+
+        Scanner& scanner;
+
+        JUCE_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (ScanJob)
+    };
 
     JUCE_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (Scanner)
+
 };
 
 void PluginListComponent::scanFor (AudioPluginFormat* format)
 {
     if (format != nullptr)
-        currentScanner = new Scanner (*this, *format, propertiesToUse, scanOnBackgroundThread);
+        currentScanner = new Scanner (*this, *format, propertiesToUse, numThreads);
 }
 
 void PluginListComponent::scanFinished (const StringArray& failedFiles)

modules/juce_audio_processors/scanning/juce_PluginListComponent.h

@@ -61,8 +61,10 @@ public:
     /** Changes the text in the panel's button. */
     void setOptionsButtonText (const String& newText);
 
-    /** Chooses whether to use the message thread or a background thread for scanning. */
-    void setScansOnMessageThread (bool useMessageThread) noexcept;
+    /** Sets how many threads to simultaneously scan for plugins.
+        If this is 0, then all scanning happens on the message thread (this is the default)
+    */
+    void setNumberOfThreadsForScanning (int numThreads);
 
     //==============================================================================
     /** @internal */
@@ -86,7 +88,7 @@ private:
     ListBox listBox;
     TextButton optionsButton;
     PropertiesFile* propertiesToUse;
-    bool scanOnBackgroundThread;
+    int numThreads;
 
     class Scanner;
     friend class Scanner;