Fix VST2 channel layout issues in Plogue Bidule, Bitwig and Ableton Live

modules/juce_audio_plugin_client/VST/juce_VST_Wrapper.cpp

@@ -276,27 +276,33 @@ public:
         #endif
          hostWindow (0)
     {
+        int maxNumInChannels, maxNumOutChannels;
+
         busUtils.findAllCompatibleLayouts();
 
         // VST-2 does not support disabling buses: so always enable all of them
         if (busUtils.hasDynamicInBuses() || busUtils.hasDynamicOutBuses())
             busUtils.enableAllBuses();
 
-        int maxNumInChannels  = busUtils.getBusCount (true)  > 0 ? busUtils.getSupportedBusLayouts (true,  0).maxNumberOfChannels() : 0;
-        int maxNumOutChannels = busUtils.getBusCount (false) > 0 ? busUtils.getSupportedBusLayouts (false, 0).maxNumberOfChannels() : 0;
+        {
+            PluginBusUtilities::ScopedBusRestorer busRestorer (busUtils);
 
-        // try setting the number of channels
-        if (maxNumInChannels > 0)
-            filter->setPreferredBusArrangement (true,  0, busUtils.getDefaultLayoutForChannelNumAndBus (true,  0, maxNumInChannels));
+            maxNumInChannels  = jmin (busUtils.getBusCount (true)  > 0 ? busUtils.getSupportedBusLayouts (true,  0).maxNumberOfChannels() : 0, hostOnlySupportsStereo() ? 2 : 8);
+            maxNumOutChannels = jmin (busUtils.getBusCount (false) > 0 ? busUtils.getSupportedBusLayouts (false, 0).maxNumberOfChannels() : 0, hostOnlySupportsStereo() ? 2 : 8);
 
-        if (maxNumOutChannels > 0)
-            filter->setPreferredBusArrangement (false, 0, busUtils.getDefaultLayoutForChannelNumAndBus (false, 0, maxNumOutChannels));
+            // try setting the number of channels
+            if (maxNumInChannels > 0)
+                filter->setPreferredBusArrangement (true,  0, busUtils.getDefaultLayoutForChannelNumAndBus (true,  0, maxNumInChannels));
 
-        resetAuxChannelsToDefaultLayout (true);
-        resetAuxChannelsToDefaultLayout (false);
+            if (maxNumOutChannels > 0)
+                filter->setPreferredBusArrangement (false, 0, busUtils.getDefaultLayoutForChannelNumAndBus (false, 0, maxNumOutChannels));
+
+            resetAuxChannelsToDefaultLayout (true);
+            resetAuxChannelsToDefaultLayout (false);
 
-        const int totalNumInChannels  = busUtils.findTotalNumChannels (true);
-        const int totalNumOutChannels = busUtils.findTotalNumChannels (false);
+            maxNumInChannels = busUtils.findTotalNumChannels (true);
+            maxNumOutChannels = busUtils.findTotalNumChannels (false);
+        }
 
         filter->setRateAndBufferSizeDetails (0, 0);
         filter->setPlayHead (this);
@@ -307,8 +313,8 @@ public:
 
         setUniqueID ((int) (JucePlugin_VSTUniqueID));
 
-        setNumInputs  (totalNumInChannels);
-        setNumOutputs (totalNumOutChannels);
+        setNumInputs  (maxNumInChannels);
+        setNumOutputs (maxNumOutChannels);
 
         canProcessReplacing (true);
         canDoubleReplacing (filter->supportsDoublePrecisionProcessing());
@@ -560,7 +566,8 @@ public:
                     tmpBuffers.channels[i] = inputs[i];
 
                 {
-                    AudioBuffer<FloatType> chans (tmpBuffers.channels, jmax (numIn, numOut), numSamples);
+                    const int numChannels = jmax (filter->getTotalNumInputChannels(), filter->getTotalNumOutputChannels());
+                    AudioBuffer<FloatType> chans (tmpBuffers.channels, numChannels, numSamples);
 
                     if (isBypassed)
                         filter->processBlockBypassed (chans, midiEvents);
@@ -657,14 +664,8 @@ public:
         {
             isProcessing = true;
 
-            const int numInChans  = filter->busArrangement.getTotalNumInputChannels();
-            const int numOutChans = filter->busArrangement.getTotalNumOutputChannels();
-
-            setNumInputs (numInChans);
-            setNumOutputs (numOutChans);
-
-            floatTempBuffers.channels.calloc ((size_t) (numInChans + numOutChans));
-            doubleTempBuffers.channels.calloc ((size_t) (numInChans + numOutChans));
+            floatTempBuffers.channels.calloc ((size_t) (cEffect.numInputs + cEffect.numOutputs));
+            doubleTempBuffers.channels.calloc ((size_t) (cEffect.numInputs + cEffect.numOutputs));
 
             double rate = getSampleRate();
             jassert (rate > 0);
@@ -935,12 +936,15 @@ public:
            // subtract the number of channels which are used by the aux channels
             int mainNumChannels = pluginInput->numChannels - busUtils.findTotalNumChannels (true, 1);
 
-            if (mainNumChannels < 0)
+            if (mainNumChannels <= 0)
                 return false;
 
-            if (mainNumChannels == pluginInput->numChannels)
-                newType = SpeakerMappings::vstArrangementTypeToChannelSet (*pluginInput);
-            else
+            if (mainNumChannels > busUtils.getSupportedBusLayouts (true, 0).maxNumberOfChannels())
+                return false;
+
+            newType = SpeakerMappings::vstArrangementTypeToChannelSet (*pluginInput);
+
+            if (mainNumChannels != newType.size())
                 newType = AudioChannelSet::canonicalChannelSet(mainNumChannels);
 
             if (busUtils.getChannelSet (true, 0) != newType)
@@ -955,12 +959,15 @@ public:
             // subtract the number of channels which are used by the aux channels
             int mainNumChannels = pluginOutput->numChannels - busUtils.findTotalNumChannels (false, 1);
 
-            if (mainNumChannels < 0)
+            if (mainNumChannels <= 0)
                 return false;
 
-            if (mainNumChannels == pluginOutput->numChannels)
-                newType = SpeakerMappings::vstArrangementTypeToChannelSet (*pluginOutput);
-            else
+            if (mainNumChannels > busUtils.getSupportedBusLayouts (false, 0).maxNumberOfChannels())
+                return false;
+
+            newType = SpeakerMappings::vstArrangementTypeToChannelSet (*pluginOutput);
+
+            if (mainNumChannels != newType.size())
                 newType = AudioChannelSet::canonicalChannelSet(mainNumChannels);
 
             AudioChannelSet oldOutputLayout = busUtils.getChannelSet (false, 0);
@@ -970,27 +977,13 @@ public:
                 if (! filter->setPreferredBusArrangement (false, 0, newType))
                     return false;
 
-            // did this change the input layout? If yes, change it back
-            if (oldInputLayout != busUtils.getChannelSet (true, 0)
-             && (busUtils.getBusCount (true) > 1 || busUtils.getBusCount (false) > 1))
-            {
-                bool success = filter->setPreferredBusArrangement (false, 0, oldOutputLayout);
-
-                jassert (success);
-                ignoreUnused (success);
-            }
+            // did this change the input layout?
+            if (oldInputLayout != busUtils.getChannelSet (true, 0) && pluginInput != nullptr)
+                return false;
         }
 
         busRestorer.release();
-
-        const int totalNumInChannels = busUtils.findTotalNumChannels(true);
-        const int totalNumOutChannels = busUtils.findTotalNumChannels(false);
-
         filter->setRateAndBufferSizeDetails(0, 0);
-        setNumInputs (totalNumInChannels);
-        setNumOutputs(totalNumOutChannels);
-
-        ioChanged();
 
         return true;
     }
@@ -1048,11 +1041,14 @@ public:
     bool getPinProperties (VstPinProperties& properties, bool direction, int index) const
     {
         // index refers to the absolute index when combining all channels of every bus
+        if (index >= (direction ? cEffect.numInputs : cEffect.numOutputs))
+            return false;
+
         const int n = busUtils.getBusCount(direction);
         int busIdx;
         for (busIdx = 0; busIdx < n; ++busIdx)
         {
-            const int numChans = busUtils.getNumChannels(direction, busIdx);
+            const int numChans = busUtils.getNumChannels (direction, busIdx);
             if (index < numChans)
                 break;
 
@@ -1060,25 +1056,25 @@ public:
         }
 
         if (busIdx >= n)
-            return false;
+        {
+            properties.flags = kVstPinUseSpeaker;
+            properties.label[0] = 0;
+            properties.shortLabel[0] = 0;
+            properties.arrangementType = kSpeakerArrEmpty;
+
+            return true;
+        }
 
         const AudioProcessor::AudioProcessorBus& busInfo = busUtils.getFilterBus (direction).getReference (busIdx);
 
         busInfo.name.copyToUTF8 (properties.label, (size_t) (kVstMaxLabelLen - 1));
         busInfo.name.copyToUTF8 (properties.shortLabel, (size_t) (kVstMaxShortLabelLen - 1));
 
-        VstInt32 type = SpeakerMappings::channelSetToVstArrangementType (busInfo.channels);
+        properties.flags = kVstPinUseSpeaker | kVstPinIsActive;
+        properties.arrangementType = SpeakerMappings::channelSetToVstArrangementType (busInfo.channels);
 
-        if (type != kSpeakerArrEmpty)
-        {
-            properties.flags = kVstPinUseSpeaker | kVstPinIsActive;
-            properties.arrangementType = type;
-        }
-        else
-        {
-            properties.flags = 0;
-            properties.arrangementType = 0;
-        }
+        if (properties.arrangementType == kSpeakerArrEmpty)
+            properties.flags &= ~kVstPinIsActive;
 
         if (busInfo.channels.size() == 2)
             properties.flags |= kVstPinIsStereo;
@@ -1133,11 +1129,14 @@ public:
         {
             Array<AudioChannelSet::ChannelType> chans (channels.getChannelTypes());
 
+            if (channels == AudioChannelSet::disabled())
+                return kSpeakerArrEmpty;
+
             for (const Mapping* m = getMappings(); m->vst2 != kSpeakerArrEmpty; ++m)
                 if (m->matches (chans))
                     return m->vst2;
 
-            return kSpeakerArrEmpty;
+            return kSpeakerArrUserDefined;
         }
 
         static void channelSetToVstArrangement (const AudioChannelSet& channels, VstSpeakerArrangement& result)
@@ -1736,7 +1735,7 @@ private:
 
         if (filter != nullptr)
         {
-            int numChannels = filter->getTotalNumInputChannels() + filter->getTotalNumOutputChannels();
+            int numChannels = cEffect.numInputs + cEffect.numOutputs;
             tmpBuffers.tempChannels.insertMultiple (0, nullptr, numChannels);
         }
     }
@@ -1763,6 +1762,13 @@ private:
         }
     }
 
+    bool hostOnlySupportsStereo () const
+    {
+        const PluginHostType host (getHostType ());
+
+        // there are probably more hosts that need listing here
+        return host.isAbletonLive() || host.isReaper();
+    }
     //==============================================================================
     JUCE_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (JuceVSTWrapper)
 };