Add WIP Python engine.

Makefile

@@ -6,13 +6,17 @@ CXXFLAGS +=
 
 LDFLAGS +=
 
-SOURCES += $(wildcard src/*.cpp)
+SOURCES += src/Prototype.cpp
 
 DISTRIBUTABLES += res examples
 DISTRIBUTABLES += $(wildcard LICENSE*)
 
 include $(RACK_DIR)/arch.mk
 
+DUKTAPE ?= 0
+QUICKJS ?= 1
+PYTHON ?= 0
+
 # Entropia File System Watcher
 efsw := dep/lib/libefsw-static-release.a
 DEPS += $(efsw)
@@ -23,10 +27,26 @@ $(efsw):
 	cd dep && $(UNZIP) e6afbec564e2.zip
 	cd dep/SpartanJ-efsw-e6afbec564e2 && premake4 gmake
 	cd dep/SpartanJ-efsw-e6afbec564e2 && $(MAKE) -C make/* config=release efsw-static-lib
+	mkdir -p dep/lib dep/include
 	cd dep/SpartanJ-efsw-e6afbec564e2 && cp lib/libefsw-static-release.a $(DEP_PATH)/lib/
 	cd dep/SpartanJ-efsw-e6afbec564e2 && cp -R include/efsw $(DEP_PATH)/include/
 
+# Duktape
+ifeq ($(DUKTAPE), 1)
+SOURCES += src/DuktapeEngine.cpp
+duktape := dep/duktape-2.4.0/src/duktape.c
+DEPS += $(duktape)
+SOURCES += $(duktape)
+FLAGS += -Idep/duktape-2.4.0/src
+$(duktape):
+	$(WGET) "https://duktape.org/duktape-2.4.0.tar.xz"
+	$(SHA256) duktape-2.4.0.tar.xz 86a89307d1633b5cedb2c6e56dc86e92679fc34b05be551722d8cc69ab0771fc
+	cd dep && $(UNTAR) ../duktape-2.4.0.tar.xz
+endif
+
 # QuickJS
+ifeq ($(QUICKJS), 1)
+SOURCES += src/QuickJSEngine.cpp
 quickjs := dep/lib/quickjs/libquickjs.a
 DEPS += $(quickjs)
 OBJECTS += $(quickjs)
@@ -37,6 +57,41 @@ endif
 $(quickjs):
 	cd QuickJS && $(MAKE) $(QUICKJS_MAKE_FLAGS)
 	cd QuickJS && $(MAKE) $(QUICKJS_MAKE_FLAGS) install
+endif
+
+# Python
+ifeq ($(PYTHON), 1)
+SOURCES += src/PythonEngine.cpp
+python := dep/lib/libpython3.7m.so
+DEPS += $(python) $(numpy)
+# OBJECTS += $(python)
+FLAGS += -Idep/include/python3.7m
+# TODO Test these flags on all platforms
+LDFLAGS += -Ldep/lib -lpython3.7m
+LDFLAGS += -lcrypt -lpthread -ldl -lutil -lm
+$(python):
+	$(WGET) "https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tar.xz"
+	$(SHA256) Python-3.7.4.tar.xz fb799134b868199930b75f26678f18932214042639cd52b16da7fd134cd9b13f
+	cd dep && $(UNTAR) ../Python-3.7.4.tar.xz
+	cd dep/Python-3.7.4 && $(CONFIGURE) --build=$(MACHINE) --enable-shared --enable-optimizations
+	cd dep/Python-3.7.4 && $(MAKE) build_all
+	cd dep/Python-3.7.4 && $(MAKE) install
+
+numpy := dep/lib/python3.7/site-packages/numpy-1.17.2-py3.7-linux-x86_64.egg
+FLAGS += -Idep/lib/python3.7/site-packages/numpy-1.17.2-py3.7-linux-x86_64.egg/numpy/core/include
+$(numpy): $(python)
+	$(WGET) "https://github.com/numpy/numpy/releases/download/v1.17.2/numpy-1.17.2.tar.gz"
+	$(SHA256) numpy-1.17.2.tar.gz 81a4f748dcfa80a7071ad8f3d9f8edb9f8bc1f0a9bdd19bfd44fd42c02bd286c
+	cd dep && $(UNTAR) ../numpy-1.17.2.tar.gz
+	# Don't try to find an external BLAS and LAPACK library.
+	cd dep/numpy-1.17.2 && NPY_BLAS_ORDER= NPY_LAPACK_ORDER= "$(DEP_PATH)"/bin/python3.7 setup.py build -j4 install install_headers
+
+# scipy: $(numpy)
+# 	$(WGET) "https://github.com/scipy/scipy/releases/download/v1.3.1/scipy-1.3.1.tar.xz"
+# 	$(SHA256) scipy-1.3.1.tar.xz 326ffdad79f113659ed0bca80f5d0ed5e28b2e967b438bb1f647d0738073a92e
+# 	cd dep && $(UNTAR) ../scipy-1.3.1.tar.xz
+# 	cd dep/scipy-1.3.1 && "$(DEP_PATH)"/bin/python3.7 setup.py build -j4 install
+endif
 
 # # LuaJIT
 # luajit := dep/lib/luajit.a
@@ -55,20 +110,6 @@ $(quickjs):
 # 	$(SHA256) julia-1.2.0-full.tar.gz 2419b268fc5c3666dd9aeb554815fe7cf9e0e7265bc9b94a43957c31a68d9184
 # 	cd dep && $(UNTAR) ../julia-1.2.0-full.tar.gz
 
-# # Python
-# python := dep/lib/libpython3.7m.a
-# DEPS += $(python)
-# OBJECTS += $(python)
-# FLAGS += -Idep/include/python3.7m
-# LDFLAGS += -lcrypt -lpthread -ldl -lutil -lm
-# $(python):
-# 	$(WGET) "https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tar.xz"
-# 	$(SHA256) Python-3.7.4.tar.xz fb799134b868199930b75f26678f18932214042639cd52b16da7fd134cd9b13f
-# 	cd dep && $(UNTAR) ../Python-3.7.4.tar.xz
-# 	cd dep/Python-3.7.4 && $(CONFIGURE) --build=$(MACHINE) --enable-optimizations
-# 	cd dep/Python-3.7.4 && $(MAKE) build_all
-# 	cd dep/Python-3.7.4 && $(MAKE) install
-
 # # Csound
 # csound := dep/lib/libcsound.a
 # DEPS += $(csound)

src/DuktapeEngine.cpp

@@ -0,0 +1,238 @@
+#include "ScriptEngine.hpp"
+#include <duktape.h>
+
+
+struct DuktapeEngine : ScriptEngine {
+	duk_context* ctx = NULL;
+
+	~DuktapeEngine() {
+		if (ctx)
+			duk_destroy_heap(ctx);
+	}
+
+	std::string getEngineName() override {
+		return "JavaScript";
+	}
+
+	int run(const std::string& path, const std::string& script) override {
+		assert(!ctx);
+		// Create duktape context
+		ctx = duk_create_heap_default();
+		if (!ctx) {
+			setMessage("Could not create duktape context");
+			return -1;
+		}
+
+		// Initialize globals
+		// user pointer
+		duk_push_pointer(ctx, this);
+		duk_put_global_string(ctx, DUK_HIDDEN_SYMBOL("p"));
+
+		// console
+		duk_idx_t consoleIdx = duk_push_object(ctx);
+		{
+			// log
+			duk_push_c_function(ctx, native_console_log, 1);
+			duk_put_prop_string(ctx, consoleIdx, "log");
+			// info (alias for log)
+			duk_push_c_function(ctx, native_console_log, 1);
+			duk_put_prop_string(ctx, consoleIdx, "info");
+			// debug
+			duk_push_c_function(ctx, native_console_debug, 1);
+			duk_put_prop_string(ctx, consoleIdx, "debug");
+			// warn
+			duk_push_c_function(ctx, native_console_warn, 1);
+			duk_put_prop_string(ctx, consoleIdx, "warn");
+		}
+		duk_put_global_string(ctx, "console");
+
+		// display
+		duk_push_c_function(ctx, native_display, 1);
+		duk_put_global_string(ctx, "display");
+
+		// config: Set defaults
+		duk_idx_t configIdx = duk_push_object(ctx);
+		{
+			// frameDivider
+			duk_push_int(ctx, 32);
+			duk_put_prop_string(ctx, configIdx, "frameDivider");
+			// bufferSize
+			duk_push_int(ctx, 1);
+			duk_put_prop_string(ctx, configIdx, "bufferSize");
+		}
+		duk_put_global_string(ctx, "config");
+
+		// Compile string
+		duk_push_string(ctx, path.c_str());
+		if (duk_pcompile_lstring_filename(ctx, 0, script.c_str(), script.size()) != 0) {
+			const char* s = duk_safe_to_string(ctx, -1);
+			rack::WARN("duktape: %s", s);
+			setMessage(s);
+			duk_pop(ctx);
+			return -1;
+		}
+		// Execute function
+		if (duk_pcall(ctx, 0)) {
+			const char* s = duk_safe_to_string(ctx, -1);
+			rack::WARN("duktape: %s", s);
+			setMessage(s);
+			duk_pop(ctx);
+			return -1;
+		}
+		// Ignore return value
+		duk_pop(ctx);
+
+		// config: Read values
+		duk_get_global_string(ctx, "config");
+		{
+			// frameDivider
+			duk_get_prop_string(ctx, -1, "frameDivider");
+			setFrameDivider(duk_get_int(ctx, -1));
+			duk_pop(ctx);
+			// bufferSize
+			duk_get_prop_string(ctx, -1, "bufferSize");
+			block->bufferSize = duk_get_int(ctx, -1);
+			duk_pop(ctx);
+		}
+		duk_pop(ctx);
+
+		// Keep process function on stack for faster calling
+		duk_get_global_string(ctx, "process");
+		if (!duk_is_function(ctx, -1)) {
+			setMessage("No process() function");
+			return -1;
+		}
+
+		// block (keep on stack)
+		duk_idx_t blockIdx = duk_push_object(ctx);
+		{
+			// inputs
+			duk_idx_t inputsIdx = duk_push_array(ctx);
+			for (int i = 0; i < NUM_ROWS; i++) {
+				duk_push_external_buffer(ctx);
+				duk_config_buffer(ctx, -1, block->inputs[i], sizeof(float) * block->bufferSize);
+				duk_push_buffer_object(ctx, -1, 0, sizeof(float) * block->bufferSize, DUK_BUFOBJ_FLOAT32ARRAY);
+				duk_put_prop_index(ctx, inputsIdx, i);
+				duk_pop(ctx);
+			}
+			duk_put_prop_string(ctx, blockIdx, "inputs");
+
+			// outputs
+			duk_idx_t outputsIdx = duk_push_array(ctx);
+			for (int i = 0; i < NUM_ROWS; i++) {
+				duk_push_external_buffer(ctx);
+				duk_config_buffer(ctx, -1, block->outputs[i], sizeof(float) * block->bufferSize);
+				duk_push_buffer_object(ctx, -1, 0, sizeof(float) * block->bufferSize, DUK_BUFOBJ_FLOAT32ARRAY);
+				duk_put_prop_index(ctx, outputsIdx, i);
+				duk_pop(ctx);
+			}
+			duk_put_prop_string(ctx, blockIdx, "outputs");
+
+			// knobs
+			duk_push_external_buffer(ctx);
+			duk_config_buffer(ctx, -1, block->knobs, sizeof(float) * NUM_ROWS);
+			duk_push_buffer_object(ctx, -1, 0, sizeof(float) * NUM_ROWS, DUK_BUFOBJ_FLOAT32ARRAY);
+			duk_put_prop_string(ctx, blockIdx, "knobs");
+			duk_pop(ctx);
+
+			// switches
+			duk_push_external_buffer(ctx);
+			duk_config_buffer(ctx, -1, block->switches, sizeof(bool) * NUM_ROWS);
+			duk_push_buffer_object(ctx, -1, 0, sizeof(bool) * NUM_ROWS, DUK_BUFOBJ_UINT8ARRAY);
+			duk_put_prop_string(ctx, blockIdx, "switches");
+			duk_pop(ctx);
+
+			// lights
+			duk_idx_t lightsIdx = duk_push_array(ctx);
+			for (int i = 0; i < NUM_ROWS; i++) {
+				duk_push_external_buffer(ctx);
+				duk_config_buffer(ctx, -1, block->lights[i], sizeof(float) * 3);
+				duk_push_buffer_object(ctx, -1, 0, sizeof(float) * 3, DUK_BUFOBJ_FLOAT32ARRAY);
+				duk_put_prop_index(ctx, lightsIdx, i);
+				duk_pop(ctx);
+			}
+			duk_put_prop_string(ctx, blockIdx, "lights");
+
+			// switchLights
+			duk_idx_t switchLightsIdx = duk_push_array(ctx);
+			for (int i = 0; i < NUM_ROWS; i++) {
+				duk_push_external_buffer(ctx);
+				duk_config_buffer(ctx, -1, block->switchLights[i], sizeof(float) * 3);
+				duk_push_buffer_object(ctx, -1, 0, sizeof(float) * 3, DUK_BUFOBJ_FLOAT32ARRAY);
+				duk_put_prop_index(ctx, switchLightsIdx, i);
+				duk_pop(ctx);
+			}
+			duk_put_prop_string(ctx, blockIdx, "switchLights");
+		}
+
+		return 0;
+	}
+
+	int process() override {
+		// block
+		duk_idx_t blockIdx = duk_get_top(ctx) - 1;
+		{
+			// sampleRate
+			duk_push_number(ctx, block->sampleRate);
+			duk_put_prop_string(ctx, blockIdx, "sampleRate");
+
+			// sampleTime
+			duk_push_number(ctx, block->sampleTime);
+			duk_put_prop_string(ctx, blockIdx, "sampleTime");
+
+			// bufferSize
+			duk_push_int(ctx, block->bufferSize);
+			duk_put_prop_string(ctx, blockIdx, "bufferSize");
+		}
+
+		// Duplicate process function
+		duk_dup(ctx, -2);
+		// Duplicate block object
+		duk_dup(ctx, -2);
+		// Call process function
+		if (duk_pcall(ctx, 1)) {
+			const char* s = duk_safe_to_string(ctx, -1);
+			rack::WARN("duktape: %s", s);
+			setMessage(s);
+			duk_pop(ctx);
+			return -1;
+		}
+		// return value
+		duk_pop(ctx);
+
+		return 0;
+	}
+
+	static DuktapeEngine* getDuktapeEngine(duk_context* ctx) {
+		duk_get_global_string(ctx, DUK_HIDDEN_SYMBOL("p"));
+		DuktapeEngine* engine = (DuktapeEngine*) duk_get_pointer(ctx, -1);
+		duk_pop(ctx);
+		return engine;
+	}
+
+	static duk_ret_t native_console_log(duk_context* ctx) {
+		const char* s = duk_safe_to_string(ctx, -1);
+		rack::INFO("VCV Prototype: %s", s);
+		return 0;
+	}
+	static duk_ret_t native_console_debug(duk_context* ctx) {
+		const char* s = duk_safe_to_string(ctx, -1);
+		rack::DEBUG("VCV Prototype: %s", s);
+		return 0;
+	}
+	static duk_ret_t native_console_warn(duk_context* ctx) {
+		const char* s = duk_safe_to_string(ctx, -1);
+		rack::WARN("VCV Prototype: %s", s);
+		return 0;
+	}
+	static duk_ret_t native_display(duk_context* ctx) {
+		const char* s = duk_safe_to_string(ctx, -1);
+		getDuktapeEngine(ctx)->setMessage(s);
+		return 0;
+	}
+};
+
+
+ScriptEngine* createDuktapeEngine() {
+	return new DuktapeEngine;
+}

src/Prototype.cpp

@@ -193,6 +193,7 @@ struct Prototype : Module {
 
 		if (script == "")
 			return;
+		this->script = script;
 
 		// Create script engine from path extension
 		std::string ext = string::filenameExtension(string::filename(path));
@@ -212,7 +213,6 @@ struct Prototype : Module {
 			return;
 		}
 		block->bufferSize = clamp(block->bufferSize, 1, MAX_BUFFER_SIZE);
-		this->script = script;
 		this->engineName = scriptEngine->getEngineName();
 	}
 

src/PythonEngine.cpp

@@ -0,0 +1,182 @@
+extern "C" {
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
+#include <numpy/arrayobject.h>
+}
+#include <dlfcn.h>
+#include "ScriptEngine.hpp"
+#include <thread>
+
+
+extern rack::Plugin* pluginInstance;
+
+
+static void initPython() {
+	if (Py_IsInitialized())
+		return;
+
+	// Okay, this is an IQ 200 solution for fixing the following issue.
+	// - Rack (the "application") `dlopen()`s this plugin with RTLD_LOCAL.
+	// - This plugin links with libpython, either statically or dynamically. In either case, symbols are hidden to "outside" libraries.
+	// - A Python script runs `import math` for example, which loads math.cpython*.so.
+	// - Since that's an "outside" library, it can't access libpython symbols, because it doesn't link to libpython itself.
+	// The best solution I have is to dlopen() with RTLD_GLOBAL within the plugin, which will make all libpython symbols available to the *entire* Rack application.
+	// The plugin still needs to -lpython because otherwise Rack will complain that there are unresolved symbols in the plugin, so after the following lines, libpython will be in memory *twice*, unless dlopen() is doing some optimization I'm not aware of.
+	std::string libDir = rack::asset::plugin(pluginInstance, "dep/lib");
+	std::string pythonLib = libDir + "/libpython3.7m.so";
+	void* handle = dlopen(pythonLib.c_str(), RTLD_NOW | RTLD_GLOBAL);
+	assert(handle);
+	// Set python path
+	std::string sep = ":";
+	std::string pythonPath = libDir + "/python3.7";
+	pythonPath += sep + libDir + "/python3.7/lib-dynload";
+	wchar_t* pythonPathW = Py_DecodeLocale(pythonPath.c_str(), NULL);
+	Py_SetPath(pythonPathW);
+	PyMem_RawFree(pythonPathW);
+	// Initialize but don't register signal handlers
+	Py_InitializeEx(0);
+
+	// PyEval_InitThreads();
+}
+
+
+struct PythonEngine : ScriptEngine {
+	PyObject* mainDict = NULL;
+	PyObject* processFunc = NULL;
+	PyObject* blockObj = NULL;
+	PyInterpreterState* interp = NULL;
+
+	~PythonEngine() {
+		if (interp)
+			PyInterpreterState_Delete(interp);
+	}
+
+	std::string getEngineName() override {
+		return "Python";
+	}
+
+	int run(const std::string& path, const std::string& script) override {
+		initPython();
+
+		// PyThreadState* tstate = PyThreadState_Get();
+		// interp = PyInterpreterState_New();
+		// PyThreadState_Swap(tstate);
+
+		// Get globals dictionary
+		PyObject* mainModule = PyImport_AddModule("__main__");
+		assert(mainModule);
+		mainDict = PyModule_GetDict(mainModule);
+		assert(mainDict);
+
+		// Add functions to globals
+		DEFER({Py_DECREF(mainDict);});
+		static PyMethodDef native_functions[] = {
+			{"display", native_display, METH_VARARGS, ""},
+			{NULL, NULL, 0, NULL},
+		};
+		if (PyModule_AddFunctions(mainModule, native_functions)) {
+			WARN("Could not add global functions");
+			return -1;
+		}
+
+		// Compile string
+		PyObject* code = Py_CompileString(script.c_str(), path.c_str(), Py_file_input);
+		if (!code) {
+			PyErr_Print();
+			return -1;
+		}
+		DEFER({Py_DECREF(code);});
+
+		// Evaluate string
+		PyObject* result = PyEval_EvalCode(code, mainDict, mainDict);
+		if (!result) {
+			PyErr_Print();
+			return -1;
+		}
+		DEFER({Py_DECREF(result);});
+
+		// Get process function from globals
+		processFunc = PyDict_GetItemString(mainDict, "process");
+		if (!processFunc) {
+			setMessage("No process() function");
+			return -1;
+		}
+		if (!PyCallable_Check(processFunc)) {
+			setMessage("process() is not callable");
+			return -1;
+		}
+
+		// Create block
+		static PyStructSequence_Field blockFields[] = {
+			{"inputs", ""},
+			{"outputs", ""},
+			{"knobs", ""},
+			{"switches", ""},
+			{"lights", ""},
+			{"switch_lights", ""},
+			{NULL, NULL},
+		};
+		static PyStructSequence_Desc blockDesc = {"block", "", blockFields, LENGTHOF(blockFields) - 1};
+		PyTypeObject* blockType = PyStructSequence_NewType(&blockDesc);
+		assert(blockType);
+		blockObj = PyStructSequence_New(blockType);
+		assert(blockObj);
+		PyStructSequence_SetItem(blockObj, 1, PyFloat_FromDouble(42.f));
+		PyStructSequence_SetItem(blockObj, 2, PyFloat_FromDouble(42.f));
+		PyStructSequence_SetItem(blockObj, 3, PyFloat_FromDouble(42.f));
+		PyStructSequence_SetItem(blockObj, 4, PyFloat_FromDouble(42.f));
+		PyStructSequence_SetItem(blockObj, 5, PyFloat_FromDouble(42.f));
+
+		// inputs
+		// npy_intp dims[] = {NUM_ROWS, MAX_BUFFER_SIZE};
+		// PyObject* inputs = PyArray_SimpleNewFromData(2, dims, NPY_FLOAT32, block->inputs);
+		// PyStructSequence_SetItem(blockObj, 0, inputs);
+
+		return 0;
+	}
+
+	int process() override {
+		// Call process()
+		PyObject* args = PyTuple_Pack(1, blockObj);
+		assert(args);
+		DEFER({Py_DECREF(args);});
+		PyObject* processResult = PyObject_CallObject(processFunc, args);
+		if (!processResult) {
+			PyErr_Print();
+
+			// PyObject *ptype, *pvalue, *ptraceback;
+			// PyErr_Fetch(&ptype, &pvalue, &ptraceback);
+			// const char* str = PyUnicode_AsUTF8(pvalue);
+			// if (!str)
+			// 	return -1;
+
+			// setMessage(str);
+			return -1;
+		}
+		DEFER({Py_DECREF(processResult);});
+
+		// PyThreadState* tstate = PyThreadState_New(interp);
+		// PyEval_RestoreThread(tstate);
+		// PyThreadState_Clear(tstate);
+		// PyThreadState_DeleteCurrent();
+		return 0;
+	}
+
+	static PyObject* native_display(PyObject* self, PyObject* args) {
+		PyObject* msg = PyTuple_GetItem(args, 0);
+		if (!msg)
+			return NULL;
+
+		const char* msgS = PyUnicode_AsUTF8(msg);
+		DEBUG("%s", msgS);
+
+		Py_INCREF(Py_None);
+		return Py_None;
+	}
+};
+
+
+ScriptEngine* createPythonEngine() {
+	return new PythonEngine;
+}

src/ScriptEngine.hpp

@@ -50,12 +50,16 @@ struct ScriptEngine {
 // List of ScriptEngines
 
 // Add your createMyEngine() function here.
+ScriptEngine* createDuktapeEngine();
 ScriptEngine* createQuickJSEngine();
+ScriptEngine* createPythonEngine();
 
 inline ScriptEngine* createScriptEngine(std::string ext) {
 	ext = rack::string::lowercase(ext);
 	if (ext == "js")
 		return createQuickJSEngine();
+	// else if (ext == "py")
+	// 	return createPythonEngine();
 	// Add your file extension check here.
 	return NULL;
 }