Python WIP.

Makefile

@@ -15,7 +15,7 @@ include $(RACK_DIR)/arch.mk
 
 DUKTAPE ?= 0
 QUICKJS ?= 1
-PYTHON ?= 0
+PYTHON ?= 1
 
 # Entropia File System Watcher
 efsw := dep/lib/libefsw-static-release.a
@@ -62,13 +62,17 @@ endif
 # Python
 ifeq ($(PYTHON), 1)
 SOURCES += src/PythonEngine.cpp
-python := dep/lib/libpython3.7m.so
+# Note this is a dynamic library, not static.
+python := dep/lib/libpython3.7m.so.1.0
 DEPS += $(python) $(numpy)
-# OBJECTS += $(python)
 FLAGS += -Idep/include/python3.7m
 # TODO Test these flags on all platforms
+# Make dynamic linker look in the plugin folder for libpython.
+LDFLAGS += -Wl,-rpath,'$$ORIGIN'/dep/lib
 LDFLAGS += -Ldep/lib -lpython3.7m
 LDFLAGS += -lcrypt -lpthread -ldl -lutil -lm
+DISTRIBUTABLES += $(python)
+DISTRIBUTABLES += dep/lib/python3.7
 $(python):
 	$(WGET) "https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tar.xz"
 	$(SHA256) Python-3.7.4.tar.xz fb799134b868199930b75f26678f18932214042639cd52b16da7fd134cd9b13f
@@ -84,7 +88,7 @@ $(numpy): $(python)
 	$(SHA256) numpy-1.17.2.tar.gz 81a4f748dcfa80a7071ad8f3d9f8edb9f8bc1f0a9bdd19bfd44fd42c02bd286c
 	cd dep && $(UNTAR) ../numpy-1.17.2.tar.gz
 	# Don't try to find an external BLAS and LAPACK library.
-	cd dep/numpy-1.17.2 && NPY_BLAS_ORDER= NPY_LAPACK_ORDER= "$(DEP_PATH)"/bin/python3.7 setup.py build -j4 install install_headers
+	cd dep/numpy-1.17.2 && LD_LIBRARY_PATH=../lib NPY_BLAS_ORDER= NPY_LAPACK_ORDER= "$(DEP_PATH)"/bin/python3.7 setup.py build -j4 install
 
 # scipy: $(numpy)
 # 	$(WGET) "https://github.com/scipy/scipy/releases/download/v1.3.1/scipy-1.3.1.tar.xz"

README.md

@@ -63,11 +63,11 @@ function process(block) {
 	*/
 	block.inputs[i][bufferIndex] // 0.0
 
-	/** Voltage of the output port of column `i`. Writable.
+	/** Voltage of the output port of column `i`. Read/write.
 	*/
 	block.outputs[i][bufferIndex] // 0.0
 
-	/** Value of the knob of column `i`. Between 0 and 1. Writable.
+	/** Value of the knob of column `i`. Between 0 and 1. Read/write.
 	*/
 	block.knobs[i] // 0.0
 
@@ -75,13 +75,13 @@ function process(block) {
 	*/
 	block.switches[i] // false
 
-	/** Brightness of the RGB LED of column `i`, between 0 and 1. Writable.
+	/** Brightness of the RGB LED of column `i`, between 0 and 1. Read/write.
 	*/
 	block.lights[i][0] // 0.0 (red)
 	block.lights[i][1] // 0.0 (green)
 	block.lights[i][2] // 0.0 (blue)
 
-	/** Brightness of the switch RGB LED of column `i`. Writable.
+	/** Brightness of the switch RGB LED of column `i`. Read/write.
 	*/
 	block.switchLights[i][0] // 0.0 (red)
 	block.switchLights[i][1] // 0.0 (green)

src/DuktapeEngine.cpp

@@ -66,7 +66,7 @@ struct DuktapeEngine : ScriptEngine {
 		duk_push_string(ctx, path.c_str());
 		if (duk_pcompile_lstring_filename(ctx, 0, script.c_str(), script.size()) != 0) {
 			const char* s = duk_safe_to_string(ctx, -1);
-			rack::WARN("duktape: %s", s);
+			WARN("duktape: %s", s);
 			setMessage(s);
 			duk_pop(ctx);
 			return -1;
@@ -74,7 +74,7 @@ struct DuktapeEngine : ScriptEngine {
 		// Execute function
 		if (duk_pcall(ctx, 0)) {
 			const char* s = duk_safe_to_string(ctx, -1);
-			rack::WARN("duktape: %s", s);
+			WARN("duktape: %s", s);
 			setMessage(s);
 			duk_pop(ctx);
 			return -1;
@@ -192,7 +192,7 @@ struct DuktapeEngine : ScriptEngine {
 		// Call process function
 		if (duk_pcall(ctx, 1)) {
 			const char* s = duk_safe_to_string(ctx, -1);
-			rack::WARN("duktape: %s", s);
+			WARN("duktape: %s", s);
 			setMessage(s);
 			duk_pop(ctx);
 			return -1;
@@ -212,17 +212,17 @@ struct DuktapeEngine : ScriptEngine {
 
 	static duk_ret_t native_console_log(duk_context* ctx) {
 		const char* s = duk_safe_to_string(ctx, -1);
-		rack::INFO("VCV Prototype: %s", s);
+		INFO("VCV Prototype: %s", s);
 		return 0;
 	}
 	static duk_ret_t native_console_debug(duk_context* ctx) {
 		const char* s = duk_safe_to_string(ctx, -1);
-		rack::DEBUG("VCV Prototype: %s", s);
+		DEBUG("VCV Prototype: %s", s);
 		return 0;
 	}
 	static duk_ret_t native_console_warn(duk_context* ctx) {
 		const char* s = duk_safe_to_string(ctx, -1);
-		rack::WARN("VCV Prototype: %s", s);
+		WARN("VCV Prototype: %s", s);
 		return 0;
 	}
 	static duk_ret_t native_display(duk_context* ctx) {

src/Prototype.cpp

@@ -479,7 +479,6 @@ struct PrototypeWidget : ModuleWidget {
 	}
 };
 
-
 void init(Plugin* p) {
 	pluginInstance = p;
 

src/PythonEngine.cpp

@@ -16,28 +16,24 @@ static void initPython() {
 	if (Py_IsInitialized())
 		return;
 
-	// Okay, this is an IQ 200 solution for fixing the following issue.
-	// - Rack (the "application") `dlopen()`s this plugin with RTLD_LOCAL.
-	// - This plugin links with libpython, either statically or dynamically. In either case, symbols are hidden to "outside" libraries.
-	// - A Python script runs `import math` for example, which loads math.cpython*.so.
-	// - Since that's an "outside" library, it can't access libpython symbols, because it doesn't link to libpython itself.
-	// The best solution I have is to dlopen() with RTLD_GLOBAL within the plugin, which will make all libpython symbols available to the *entire* Rack application.
-	// The plugin still needs to -lpython because otherwise Rack will complain that there are unresolved symbols in the plugin, so after the following lines, libpython will be in memory *twice*, unless dlopen() is doing some optimization I'm not aware of.
-	std::string libDir = rack::asset::plugin(pluginInstance, "dep/lib");
-	std::string pythonLib = libDir + "/libpython3.7m.so";
-	void* handle = dlopen(pythonLib.c_str(), RTLD_NOW | RTLD_GLOBAL);
-	assert(handle);
+	std::string pythonDir = rack::asset::plugin(pluginInstance, "dep/lib/python3.7");
 	// Set python path
 	std::string sep = ":";
-	std::string pythonPath = libDir + "/python3.7";
-	pythonPath += sep + libDir + "/python3.7/lib-dynload";
+	std::string pythonPath = pythonDir;
+	pythonPath += sep + pythonDir + "/lib-dynload";
+	// TODO Don't install to egg
+	pythonPath += sep + pythonDir + "/site-packages/numpy-1.17.2-py3.7-linux-x86_64.egg";
 	wchar_t* pythonPathW = Py_DecodeLocale(pythonPath.c_str(), NULL);
 	Py_SetPath(pythonPathW);
 	PyMem_RawFree(pythonPathW);
 	// Initialize but don't register signal handlers
 	Py_InitializeEx(0);
+	assert(Py_IsInitialized());
 
-	// PyEval_InitThreads();
+	PyEval_InitThreads();
+
+	// Import numpy
+	assert(_import_array() == 0);
 }
 
 
@@ -48,6 +44,12 @@ struct PythonEngine : ScriptEngine {
 	PyInterpreterState* interp = NULL;
 
 	~PythonEngine() {
+		if (mainDict)
+			Py_DECREF(mainDict);
+		if (processFunc)
+			Py_DECREF(processFunc);
+		if (blockObj)
+			Py_DECREF(blockObj);
 		if (interp)
 			PyInterpreterState_Delete(interp);
 	}
@@ -66,13 +68,17 @@ struct PythonEngine : ScriptEngine {
 		// Get globals dictionary
 		PyObject* mainModule = PyImport_AddModule("__main__");
 		assert(mainModule);
+		DEFER({Py_DECREF(mainModule);});
 		mainDict = PyModule_GetDict(mainModule);
 		assert(mainDict);
 
+		// Set context pointer
+		PyObject* thisO = PyCapsule_New(this, NULL, NULL);
+		PyDict_SetItemString(mainDict, "this", thisO);
+
 		// Add functions to globals
-		DEFER({Py_DECREF(mainDict);});
 		static PyMethodDef native_functions[] = {
-			{"display", native_display, METH_VARARGS, ""},
+			{"display", nativeDisplay, METH_VARARGS, ""},
 			{NULL, NULL, 0, NULL},
 		};
 		if (PyModule_AddFunctions(mainModule, native_functions)) {
@@ -96,17 +102,6 @@ struct PythonEngine : ScriptEngine {
 		}
 		DEFER({Py_DECREF(result);});
 
-		// Get process function from globals
-		processFunc = PyDict_GetItemString(mainDict, "process");
-		if (!processFunc) {
-			setMessage("No process() function");
-			return -1;
-		}
-		if (!PyCallable_Check(processFunc)) {
-			setMessage("process() is not callable");
-			return -1;
-		}
-
 		// Create block
 		static PyStructSequence_Field blockFields[] = {
 			{"inputs", ""},
@@ -117,26 +112,61 @@ struct PythonEngine : ScriptEngine {
 			{"switch_lights", ""},
 			{NULL, NULL},
 		};
-		static PyStructSequence_Desc blockDesc = {"block", "", blockFields, LENGTHOF(blockFields) - 1};
+		static PyStructSequence_Desc blockDesc = {"Block", "", blockFields, LENGTHOF(blockFields) - 1};
 		PyTypeObject* blockType = PyStructSequence_NewType(&blockDesc);
 		assert(blockType);
+		DEBUG("ref %d", Py_REFCNT(blockType));
+
 		blockObj = PyStructSequence_New(blockType);
 		assert(blockObj);
-		PyStructSequence_SetItem(blockObj, 1, PyFloat_FromDouble(42.f));
-		PyStructSequence_SetItem(blockObj, 2, PyFloat_FromDouble(42.f));
-		PyStructSequence_SetItem(blockObj, 3, PyFloat_FromDouble(42.f));
-		PyStructSequence_SetItem(blockObj, 4, PyFloat_FromDouble(42.f));
-		PyStructSequence_SetItem(blockObj, 5, PyFloat_FromDouble(42.f));
+		DEBUG("ref %d", Py_REFCNT(blockObj));
 
 		// inputs
-		// npy_intp dims[] = {NUM_ROWS, MAX_BUFFER_SIZE};
-		// PyObject* inputs = PyArray_SimpleNewFromData(2, dims, NPY_FLOAT32, block->inputs);
-		// PyStructSequence_SetItem(blockObj, 0, inputs);
+		npy_intp inputsDims[] = {NUM_ROWS, MAX_BUFFER_SIZE};
+		PyObject* inputs = PyArray_SimpleNewFromData(2, inputsDims, NPY_FLOAT32, block->inputs);
+		PyStructSequence_SetItem(blockObj, 0, inputs);
+
+		// outputs
+		npy_intp outputsDims[] = {NUM_ROWS, MAX_BUFFER_SIZE};
+		PyObject* outputs = PyArray_SimpleNewFromData(2, outputsDims, NPY_FLOAT32, block->outputs);
+		PyStructSequence_SetItem(blockObj, 1, outputs);
+
+		// knobs
+		npy_intp knobsDims[] = {NUM_ROWS};
+		PyObject* knobs = PyArray_SimpleNewFromData(1, knobsDims, NPY_FLOAT32, block->knobs);
+		PyStructSequence_SetItem(blockObj, 2, knobs);
+
+		// switches
+		npy_intp switchesDims[] = {NUM_ROWS};
+		PyObject* switches = PyArray_SimpleNewFromData(1, switchesDims, NPY_BOOL, block->switches);
+		PyStructSequence_SetItem(blockObj, 3, switches);
+
+		// lights
+		npy_intp lightsDims[] = {NUM_ROWS, 3};
+		PyObject* lights = PyArray_SimpleNewFromData(2, lightsDims, NPY_FLOAT32, block->lights);
+		PyStructSequence_SetItem(blockObj, 4, lights);
+
+		// switchLights
+		npy_intp switchLightsDims[] = {NUM_ROWS, 3};
+		PyObject* switchLights = PyArray_SimpleNewFromData(2, switchLightsDims, NPY_FLOAT32, block->switchLights);
+		PyStructSequence_SetItem(blockObj, 5, switchLights);
+
+		// Get process function from globals
+		processFunc = PyDict_GetItemString(mainDict, "process");
+		if (!processFunc) {
+			setMessage("No process() function");
+			return -1;
+		}
+		if (!PyCallable_Check(processFunc)) {
+			setMessage("process() is not callable");
+			return -1;
+		}
 
 		return 0;
 	}
 
 	int process() override {
+		// DEBUG("ref %d", Py_REFCNT(blockObj));
 		// Call process()
 		PyObject* args = PyTuple_Pack(1, blockObj);
 		assert(args);
@@ -163,13 +193,23 @@ struct PythonEngine : ScriptEngine {
 		return 0;
 	}
 
-	static PyObject* native_display(PyObject* self, PyObject* args) {
-		PyObject* msg = PyTuple_GetItem(args, 0);
-		if (!msg)
+	static PyObject* nativeDisplay(PyObject* self, PyObject* args) {
+		PyObject* mainDict = PyEval_GetGlobals();
+		assert(mainDict);
+		PyObject* thatO = PyDict_GetItemString(mainDict, "this");
+		assert(thatO);
+		PythonEngine* that = (PythonEngine*) PyCapsule_GetPointer(thatO, NULL);
+		assert(that);
+
+		PyObject* msgO = PyTuple_GetItem(args, 0);
+		if (!msgO)
 			return NULL;
 
-		const char* msgS = PyUnicode_AsUTF8(msg);
-		DEBUG("%s", msgS);
+		PyObject* msgS = PyObject_Str(msgO);
+		DEFER({Py_DECREF(msgS);});
+
+		const char* msg = PyUnicode_AsUTF8(msgS);
+		that->setMessage(msg);
 
 		Py_INCREF(Py_None);
 		return Py_None;

src/QuickJSEngine.cpp

@@ -142,7 +142,7 @@ struct QuickJSEngine : ScriptEngine {
       for (int i = 0; i < NUM_ROWS; i++) {
         JSValue buffer = JS_NewArrayBuffer(ctx, (uint8_t *) block->inputs[i], sizeof(float) * block->bufferSize, NULL, NULL, true);
         if (JS_SetPropertyUint32(ctx, arr, i, buffer) < 0) {
-          rack::WARN("Unable to set property %d of inputs array", i);
+          WARN("Unable to set property %d of inputs array", i);
         }
       }
       JS_SetPropertyStr(ctx, blockIdx, "inputs", arr);
@@ -152,7 +152,7 @@ struct QuickJSEngine : ScriptEngine {
       for (int i = 0; i < NUM_ROWS; i++) {
         JSValue buffer = JS_NewArrayBuffer(ctx, (uint8_t *) block->outputs[i], sizeof(float) * block->bufferSize, NULL, NULL, true);
         if (JS_SetPropertyUint32(ctx, arr, i, buffer) < 0) {
-          rack::WARN("Unable to set property %d of outputs array", i);
+          WARN("Unable to set property %d of outputs array", i);
         }
       }
       JS_SetPropertyStr(ctx, blockIdx, "outputs", arr);
@@ -170,7 +170,7 @@ struct QuickJSEngine : ScriptEngine {
       for (int i = 0; i < NUM_ROWS; i++) {
         JSValue buffer = JS_NewArrayBuffer(ctx, (uint8_t *) &block->lights[i], sizeof(float) * 3, NULL, NULL, true);
         if (JS_SetPropertyUint32(ctx, arr, i, buffer) < 0) {
-          rack::WARN("Unable to set property %d of lights array", i);
+          WARN("Unable to set property %d of lights array", i);
         }
       }
       JS_SetPropertyStr(ctx, blockIdx, "lights", arr);
@@ -180,7 +180,7 @@ struct QuickJSEngine : ScriptEngine {
       for (int i = 0; i < NUM_ROWS; i++) {
         JSValue buffer = JS_NewArrayBuffer(ctx, (uint8_t *) &block->switchLights[i], sizeof(float) * 3, NULL, NULL, true);
         if (JS_SetPropertyUint32(ctx, arr, i, buffer) < 0) {
-          rack::WARN("Unable to set property %d of switchLights array", i);
+          WARN("Unable to set property %d of switchLights array", i);
         }
       }
       JS_SetPropertyStr(ctx, blockIdx, "switchLights", arr);
@@ -203,7 +203,7 @@ struct QuickJSEngine : ScriptEngine {
     JSValue hack = JS_Eval(ctx, updateTypes.c_str(), updateTypes.size(), "QuickJS Hack", 0);
     if (JS_IsException(hack)) {
       std::string errorString = ErrorToString(ctx);
-      rack::WARN("QuickJS: %s", errorString.c_str());
+      WARN("QuickJS: %s", errorString.c_str());
       setMessage(errorString.c_str());
     }
 
@@ -214,7 +214,7 @@ struct QuickJSEngine : ScriptEngine {
 
     if (JS_IsException(val)) {
       std::string errorString = ErrorToString(ctx);
-      rack::WARN("QuickJS: %s", errorString.c_str());
+      WARN("QuickJS: %s", errorString.c_str());
       setMessage(errorString.c_str());
 
       JS_FreeValue(ctx, val);
@@ -253,7 +253,7 @@ struct QuickJSEngine : ScriptEngine {
 
     if (JS_IsException(val)) {
       std::string errorString = ErrorToString(ctx);
-      rack::WARN("QuickJS: %s", errorString.c_str());
+      WARN("QuickJS: %s", errorString.c_str());
       setMessage(errorString.c_str());
 
       JS_FreeValue(ctx, val);
@@ -286,7 +286,7 @@ struct QuickJSEngine : ScriptEngine {
                                     int argc, JSValueConst *argv) {
     if (argc) {
       const char *s = JS_ToCString(ctx, argv[0]);
-      rack::INFO("VCV Prototype: %s", s);
+      INFO("VCV Prototype: %s", s);
     }
 		return JS_UNDEFINED;
 	}
@@ -294,7 +294,7 @@ struct QuickJSEngine : ScriptEngine {
                                     int argc, JSValueConst *argv) {
     if (argc) {
       const char *s = JS_ToCString(ctx, argv[0]);
-      rack::DEBUG("VCV Prototype: %s", s);
+      DEBUG("VCV Prototype: %s", s);
     }
 		return JS_UNDEFINED;
 	}
@@ -302,7 +302,7 @@ struct QuickJSEngine : ScriptEngine {
                                     int argc, JSValueConst *argv) {
     if (argc) {
       const char *s = JS_ToCString(ctx, argv[0]);
-      rack::INFO("VCV Prototype: %s", s);
+      INFO("VCV Prototype: %s", s);
     }
 		return JS_UNDEFINED;
 	}
@@ -310,7 +310,7 @@ struct QuickJSEngine : ScriptEngine {
                                     int argc, JSValueConst *argv) {
     if (argc) {
       const char *s = JS_ToCString(ctx, argv[0]);
-      rack::WARN("VCV Prototype: %s", s);
+      WARN("VCV Prototype: %s", s);
     }
 		return JS_UNDEFINED;
 	}

src/ScriptEngine.hpp

@@ -58,8 +58,8 @@ inline ScriptEngine* createScriptEngine(std::string ext) {
 	ext = rack::string::lowercase(ext);
 	if (ext == "js")
 		return createQuickJSEngine();
-	// else if (ext == "py")
-	// 	return createPythonEngine();
+	else if (ext == "py")
+		return createPythonEngine();
 	// Add your file extension check here.
 	return NULL;
 }

tests/hello.py

@@ -0,0 +1,17 @@
+frame = 0
+
+def process(block):
+	global frame
+	frame += 1
+	print(frame)
+	# print(block)
+	# block.switch_lights[:, 2] = 1
+	print(block.inputs)
+	print(block.outputs)
+	print(block.knobs)
+	print(block.switches)
+	print(block.lights)
+	print(block.switch_lights)
+	print("===")
+
+print("Hello, world!")