Rework some vst code so that carla works inside Tracktion

source/plugin/carla-vst.cpp

@@ -534,29 +534,39 @@ private:
 
 // -----------------------------------------------------------------------
 
+struct VstObject {
+    audioMasterCallback audioMaster;
+    NativePlugin* plugin;
+};
+
 #ifdef VESTIGE_HEADER
-# define handlePtr   ((NativePlugin*)effect->ptr2)
-# define validEffect effect != nullptr && effect->ptr2 != nullptr
+# define validObject  effect != nullptr && effect->ptr3 != nullptr
+# define validPlugin  effect != nullptr && effect->ptr3 != nullptr && ((VstObject*)effect->ptr3)->plugin != nullptr
+# define vstObjectPtr (VstObject*)effect->ptr3
 #else
-# define handlePtr   ((NativePlugin*)effect->resvd2)
-# define validEffect effect != nullptr && effect->resvd2 != 0
+# define validObject  effect != nullptr && effect->object != nullptr
+# define validPlugin  effect != nullptr && effect->object != nullptr && ((VstObject*)effect->object)->plugin != nullptr
+# define vstObjectPtr (VstObject*)effect->object
 #endif
 
+#define pluginPtr     (vstObjectPtr)->plugin
+
 static intptr_t vst_dispatcherCallback(AEffect* effect, int32_t opcode, int32_t index, intptr_t value, void* ptr, float opt)
 {
     // handle base opcodes
     switch (opcode)
     {
     case effOpen:
-#ifdef VESTIGE_HEADER
-        if (effect != nullptr && effect->ptr3 != nullptr)
-        {
-            audioMasterCallback audioMaster = (audioMasterCallback)effect->ptr3;
-#else
-        if (effect != nullptr && effect->object != nullptr)
+        if (VstObject* const obj = vstObjectPtr)
         {
-            audioMasterCallback audioMaster = (audioMasterCallback)effect->object;
-#endif
+            // this must always be valid
+            CARLA_SAFE_ASSERT_RETURN(obj->audioMaster != nullptr, 0);
+
+            // some hosts call effOpen twice
+            CARLA_SAFE_ASSERT_RETURN(obj->plugin == nullptr, 1);
+
+            audioMasterCallback audioMaster = (audioMasterCallback)obj->audioMaster;
+
             d_lastBufferSize = static_cast<uint32_t>(audioMaster(effect, audioMasterGetBlockSize, 0, 0, nullptr, 0.0f));
             d_lastSampleRate = static_cast<double>(audioMaster(effect, audioMasterGetSampleRate, 0, 0, nullptr, 0.0f));
 
@@ -588,31 +598,35 @@ static intptr_t vst_dispatcherCallback(AEffect* effect, int32_t opcode, int32_t
 
             CARLA_SAFE_ASSERT_RETURN(pluginDesc != nullptr, 0);
 
-            NativePlugin* const plugin(new NativePlugin(audioMaster, effect, pluginDesc));
-#ifdef VESTIGE_HEADER
-            effect->ptr2 = plugin;
-#else
-            effect->resvd2 = (intptr_t)plugin;
-#endif
+            obj->plugin = new NativePlugin(audioMaster, effect, pluginDesc);
             return 1;
         }
         return 0;
 
     case effClose:
-        if (validEffect)
+        if (VstObject* const obj = vstObjectPtr)
         {
-#ifdef VESTIGE_HEADER
-            delete (NativePlugin*)effect->ptr2;
-            effect->ptr2 = nullptr;
+            if (obj->plugin != nullptr)
+            {
+                delete obj->plugin;
+                obj->plugin = nullptr;
+            }
+
+#if 0
+            /* This code invalidates the object created in VSTPluginMain
+             * Probably not safe against all hosts */
+            obj->audioMaster = nullptr;
+# ifdef VESTIGE_HEADER
             effect->ptr3 = nullptr;
-#else
-            delete (NativePlugin*)effect->resvd2;
-            effect->resvd2 = 0;
-            effect->object = nullptr;
+# else
+            vstObjectPtr = nullptr;
+# endif
+            delete obj;
 #endif
-            delete effect;
+
             return 1;
         }
+        //delete effect;
         return 0;
 
     case effGetPlugCategory:
@@ -662,43 +676,46 @@ static intptr_t vst_dispatcherCallback(AEffect* effect, int32_t opcode, int32_t
     };
 
     // handle advanced opcodes
-    if (validEffect)
-        return handlePtr->vst_dispatcher(opcode, index, value, ptr, opt);
+    if (validPlugin)
+        return pluginPtr->vst_dispatcher(opcode, index, value, ptr, opt);
 
     return 0;
 }
 
 static float vst_getParameterCallback(AEffect* effect, int32_t index)
 {
-    if (validEffect)
-        return handlePtr->vst_getParameter(index);
+    if (validPlugin)
+        return pluginPtr->vst_getParameter(index);
     return 0.0f;
 }
 
 static void vst_setParameterCallback(AEffect* effect, int32_t index, float value)
 {
-    if (validEffect)
-        handlePtr->vst_setParameter(index, value);
+    if (validPlugin)
+        pluginPtr->vst_setParameter(index, value);
 }
 
 static void vst_processCallback(AEffect* effect, float** inputs, float** outputs, int32_t sampleFrames)
 {
-    if (validEffect)
-        handlePtr->vst_processReplacing(const_cast<const float**>(inputs), outputs, sampleFrames);
+    if (validPlugin)
+        pluginPtr->vst_processReplacing(const_cast<const float**>(inputs), outputs, sampleFrames);
 }
 
 static void vst_processReplacingCallback(AEffect* effect, float** inputs, float** outputs, int32_t sampleFrames)
 {
-    if (validEffect)
-        handlePtr->vst_processReplacing(const_cast<const float**>(inputs), outputs, sampleFrames);
+    if (validPlugin)
+        pluginPtr->vst_processReplacing(const_cast<const float**>(inputs), outputs, sampleFrames);
 }
 
-#undef handlePtr
+#undef pluginPtr
+#undef validObject
+#undef validPlugin
+#undef vstObjectPtr
 
 // -----------------------------------------------------------------------
 
 CARLA_EXPORT
-#ifdef CARLA_OS_WIN
+#if defined(CARLA_OS_WIN) || defined(CARLA_OS_MAC)
 const AEffect* VSTPluginMain(audioMasterCallback audioMaster);
 #else
 const AEffect* VSTPluginMain(audioMasterCallback audioMaster) asm ("main");
@@ -760,10 +777,13 @@ const AEffect* VSTPluginMain(audioMasterCallback audioMaster)
     effect->processReplacing = vst_processReplacingCallback;
 
     // pointers
+    VstObject* const obj(new VstObject());
+    obj->audioMaster = audioMaster;
+    obj->plugin      = nullptr;
 #ifdef VESTIGE_HEADER
-    effect->ptr3   = (void*)audioMaster;
+    effect->ptr3   = obj;
 #else
-    effect->object = (void*)audioMaster;
+    effect->object = obj;
 #endif
 
     return effect;