From 6e3c04b952f303363c02580820dbabee3b1c0b2e Mon Sep 17 00:00:00 2001 From: Andrew Belt Date: Thu, 10 Oct 2019 14:59:19 -0400 Subject: [PATCH] Python WIP. --- Makefile | 12 +++-- README.md | 8 +-- src/DuktapeEngine.cpp | 12 ++--- src/Prototype.cpp | 1 - src/PythonEngine.cpp | 122 ++++++++++++++++++++++++++++-------------- src/QuickJSEngine.cpp | 22 ++++---- src/ScriptEngine.hpp | 4 +- tests/hello.py | 17 ++++++ 8 files changed, 129 insertions(+), 69 deletions(-) create mode 100644 tests/hello.py diff --git a/Makefile b/Makefile index 465ec06..39a75da 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ include $(RACK_DIR)/arch.mk DUKTAPE ?= 0 QUICKJS ?= 1 -PYTHON ?= 0 +PYTHON ?= 1 # Entropia File System Watcher efsw := dep/lib/libefsw-static-release.a @@ -62,13 +62,17 @@ endif # Python ifeq ($(PYTHON), 1) SOURCES += src/PythonEngine.cpp -python := dep/lib/libpython3.7m.so +# Note this is a dynamic library, not static. +python := dep/lib/libpython3.7m.so.1.0 DEPS += $(python) $(numpy) -# OBJECTS += $(python) FLAGS += -Idep/include/python3.7m # TODO Test these flags on all platforms +# Make dynamic linker look in the plugin folder for libpython. +LDFLAGS += -Wl,-rpath,'$$ORIGIN'/dep/lib LDFLAGS += -Ldep/lib -lpython3.7m LDFLAGS += -lcrypt -lpthread -ldl -lutil -lm +DISTRIBUTABLES += $(python) +DISTRIBUTABLES += dep/lib/python3.7 $(python): $(WGET) "https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tar.xz" $(SHA256) Python-3.7.4.tar.xz fb799134b868199930b75f26678f18932214042639cd52b16da7fd134cd9b13f @@ -84,7 +88,7 @@ $(numpy): $(python) $(SHA256) numpy-1.17.2.tar.gz 81a4f748dcfa80a7071ad8f3d9f8edb9f8bc1f0a9bdd19bfd44fd42c02bd286c cd dep && $(UNTAR) ../numpy-1.17.2.tar.gz # Don't try to find an external BLAS and LAPACK library. - cd dep/numpy-1.17.2 && NPY_BLAS_ORDER= NPY_LAPACK_ORDER= "$(DEP_PATH)"/bin/python3.7 setup.py build -j4 install install_headers + cd dep/numpy-1.17.2 && LD_LIBRARY_PATH=../lib NPY_BLAS_ORDER= NPY_LAPACK_ORDER= "$(DEP_PATH)"/bin/python3.7 setup.py build -j4 install # scipy: $(numpy) # $(WGET) "https://github.com/scipy/scipy/releases/download/v1.3.1/scipy-1.3.1.tar.xz" diff --git a/README.md b/README.md index 66dd18c..56ad451 100644 --- a/README.md +++ b/README.md @@ -63,11 +63,11 @@ function process(block) { */ block.inputs[i][bufferIndex] // 0.0 - /** Voltage of the output port of column `i`. Writable. + /** Voltage of the output port of column `i`. Read/write. */ block.outputs[i][bufferIndex] // 0.0 - /** Value of the knob of column `i`. Between 0 and 1. Writable. + /** Value of the knob of column `i`. Between 0 and 1. Read/write. */ block.knobs[i] // 0.0 @@ -75,13 +75,13 @@ function process(block) { */ block.switches[i] // false - /** Brightness of the RGB LED of column `i`, between 0 and 1. Writable. + /** Brightness of the RGB LED of column `i`, between 0 and 1. Read/write. */ block.lights[i][0] // 0.0 (red) block.lights[i][1] // 0.0 (green) block.lights[i][2] // 0.0 (blue) - /** Brightness of the switch RGB LED of column `i`. Writable. + /** Brightness of the switch RGB LED of column `i`. Read/write. */ block.switchLights[i][0] // 0.0 (red) block.switchLights[i][1] // 0.0 (green) diff --git a/src/DuktapeEngine.cpp b/src/DuktapeEngine.cpp index 612f4a0..7cea986 100644 --- a/src/DuktapeEngine.cpp +++ b/src/DuktapeEngine.cpp @@ -66,7 +66,7 @@ struct DuktapeEngine : ScriptEngine { duk_push_string(ctx, path.c_str()); if (duk_pcompile_lstring_filename(ctx, 0, script.c_str(), script.size()) != 0) { const char* s = duk_safe_to_string(ctx, -1); - rack::WARN("duktape: %s", s); + WARN("duktape: %s", s); setMessage(s); duk_pop(ctx); return -1; @@ -74,7 +74,7 @@ struct DuktapeEngine : ScriptEngine { // Execute function if (duk_pcall(ctx, 0)) { const char* s = duk_safe_to_string(ctx, -1); - rack::WARN("duktape: %s", s); + WARN("duktape: %s", s); setMessage(s); duk_pop(ctx); return -1; @@ -192,7 +192,7 @@ struct DuktapeEngine : ScriptEngine { // Call process function if (duk_pcall(ctx, 1)) { const char* s = duk_safe_to_string(ctx, -1); - rack::WARN("duktape: %s", s); + WARN("duktape: %s", s); setMessage(s); duk_pop(ctx); return -1; @@ -212,17 +212,17 @@ struct DuktapeEngine : ScriptEngine { static duk_ret_t native_console_log(duk_context* ctx) { const char* s = duk_safe_to_string(ctx, -1); - rack::INFO("VCV Prototype: %s", s); + INFO("VCV Prototype: %s", s); return 0; } static duk_ret_t native_console_debug(duk_context* ctx) { const char* s = duk_safe_to_string(ctx, -1); - rack::DEBUG("VCV Prototype: %s", s); + DEBUG("VCV Prototype: %s", s); return 0; } static duk_ret_t native_console_warn(duk_context* ctx) { const char* s = duk_safe_to_string(ctx, -1); - rack::WARN("VCV Prototype: %s", s); + WARN("VCV Prototype: %s", s); return 0; } static duk_ret_t native_display(duk_context* ctx) { diff --git a/src/Prototype.cpp b/src/Prototype.cpp index eb96090..720eeae 100644 --- a/src/Prototype.cpp +++ b/src/Prototype.cpp @@ -479,7 +479,6 @@ struct PrototypeWidget : ModuleWidget { } }; - void init(Plugin* p) { pluginInstance = p; diff --git a/src/PythonEngine.cpp b/src/PythonEngine.cpp index 4dd05e4..31f0af1 100644 --- a/src/PythonEngine.cpp +++ b/src/PythonEngine.cpp @@ -16,28 +16,24 @@ static void initPython() { if (Py_IsInitialized()) return; - // Okay, this is an IQ 200 solution for fixing the following issue. - // - Rack (the "application") `dlopen()`s this plugin with RTLD_LOCAL. - // - This plugin links with libpython, either statically or dynamically. In either case, symbols are hidden to "outside" libraries. - // - A Python script runs `import math` for example, which loads math.cpython*.so. - // - Since that's an "outside" library, it can't access libpython symbols, because it doesn't link to libpython itself. - // The best solution I have is to dlopen() with RTLD_GLOBAL within the plugin, which will make all libpython symbols available to the *entire* Rack application. - // The plugin still needs to -lpython because otherwise Rack will complain that there are unresolved symbols in the plugin, so after the following lines, libpython will be in memory *twice*, unless dlopen() is doing some optimization I'm not aware of. - std::string libDir = rack::asset::plugin(pluginInstance, "dep/lib"); - std::string pythonLib = libDir + "/libpython3.7m.so"; - void* handle = dlopen(pythonLib.c_str(), RTLD_NOW | RTLD_GLOBAL); - assert(handle); + std::string pythonDir = rack::asset::plugin(pluginInstance, "dep/lib/python3.7"); // Set python path std::string sep = ":"; - std::string pythonPath = libDir + "/python3.7"; - pythonPath += sep + libDir + "/python3.7/lib-dynload"; + std::string pythonPath = pythonDir; + pythonPath += sep + pythonDir + "/lib-dynload"; + // TODO Don't install to egg + pythonPath += sep + pythonDir + "/site-packages/numpy-1.17.2-py3.7-linux-x86_64.egg"; wchar_t* pythonPathW = Py_DecodeLocale(pythonPath.c_str(), NULL); Py_SetPath(pythonPathW); PyMem_RawFree(pythonPathW); // Initialize but don't register signal handlers Py_InitializeEx(0); + assert(Py_IsInitialized()); - // PyEval_InitThreads(); + PyEval_InitThreads(); + + // Import numpy + assert(_import_array() == 0); } @@ -48,6 +44,12 @@ struct PythonEngine : ScriptEngine { PyInterpreterState* interp = NULL; ~PythonEngine() { + if (mainDict) + Py_DECREF(mainDict); + if (processFunc) + Py_DECREF(processFunc); + if (blockObj) + Py_DECREF(blockObj); if (interp) PyInterpreterState_Delete(interp); } @@ -66,13 +68,17 @@ struct PythonEngine : ScriptEngine { // Get globals dictionary PyObject* mainModule = PyImport_AddModule("__main__"); assert(mainModule); + DEFER({Py_DECREF(mainModule);}); mainDict = PyModule_GetDict(mainModule); assert(mainDict); + // Set context pointer + PyObject* thisO = PyCapsule_New(this, NULL, NULL); + PyDict_SetItemString(mainDict, "this", thisO); + // Add functions to globals - DEFER({Py_DECREF(mainDict);}); static PyMethodDef native_functions[] = { - {"display", native_display, METH_VARARGS, ""}, + {"display", nativeDisplay, METH_VARARGS, ""}, {NULL, NULL, 0, NULL}, }; if (PyModule_AddFunctions(mainModule, native_functions)) { @@ -96,17 +102,6 @@ struct PythonEngine : ScriptEngine { } DEFER({Py_DECREF(result);}); - // Get process function from globals - processFunc = PyDict_GetItemString(mainDict, "process"); - if (!processFunc) { - setMessage("No process() function"); - return -1; - } - if (!PyCallable_Check(processFunc)) { - setMessage("process() is not callable"); - return -1; - } - // Create block static PyStructSequence_Field blockFields[] = { {"inputs", ""}, @@ -117,26 +112,61 @@ struct PythonEngine : ScriptEngine { {"switch_lights", ""}, {NULL, NULL}, }; - static PyStructSequence_Desc blockDesc = {"block", "", blockFields, LENGTHOF(blockFields) - 1}; + static PyStructSequence_Desc blockDesc = {"Block", "", blockFields, LENGTHOF(blockFields) - 1}; PyTypeObject* blockType = PyStructSequence_NewType(&blockDesc); assert(blockType); + DEBUG("ref %d", Py_REFCNT(blockType)); + blockObj = PyStructSequence_New(blockType); assert(blockObj); - PyStructSequence_SetItem(blockObj, 1, PyFloat_FromDouble(42.f)); - PyStructSequence_SetItem(blockObj, 2, PyFloat_FromDouble(42.f)); - PyStructSequence_SetItem(blockObj, 3, PyFloat_FromDouble(42.f)); - PyStructSequence_SetItem(blockObj, 4, PyFloat_FromDouble(42.f)); - PyStructSequence_SetItem(blockObj, 5, PyFloat_FromDouble(42.f)); + DEBUG("ref %d", Py_REFCNT(blockObj)); // inputs - // npy_intp dims[] = {NUM_ROWS, MAX_BUFFER_SIZE}; - // PyObject* inputs = PyArray_SimpleNewFromData(2, dims, NPY_FLOAT32, block->inputs); - // PyStructSequence_SetItem(blockObj, 0, inputs); + npy_intp inputsDims[] = {NUM_ROWS, MAX_BUFFER_SIZE}; + PyObject* inputs = PyArray_SimpleNewFromData(2, inputsDims, NPY_FLOAT32, block->inputs); + PyStructSequence_SetItem(blockObj, 0, inputs); + + // outputs + npy_intp outputsDims[] = {NUM_ROWS, MAX_BUFFER_SIZE}; + PyObject* outputs = PyArray_SimpleNewFromData(2, outputsDims, NPY_FLOAT32, block->outputs); + PyStructSequence_SetItem(blockObj, 1, outputs); + + // knobs + npy_intp knobsDims[] = {NUM_ROWS}; + PyObject* knobs = PyArray_SimpleNewFromData(1, knobsDims, NPY_FLOAT32, block->knobs); + PyStructSequence_SetItem(blockObj, 2, knobs); + + // switches + npy_intp switchesDims[] = {NUM_ROWS}; + PyObject* switches = PyArray_SimpleNewFromData(1, switchesDims, NPY_BOOL, block->switches); + PyStructSequence_SetItem(blockObj, 3, switches); + + // lights + npy_intp lightsDims[] = {NUM_ROWS, 3}; + PyObject* lights = PyArray_SimpleNewFromData(2, lightsDims, NPY_FLOAT32, block->lights); + PyStructSequence_SetItem(blockObj, 4, lights); + + // switchLights + npy_intp switchLightsDims[] = {NUM_ROWS, 3}; + PyObject* switchLights = PyArray_SimpleNewFromData(2, switchLightsDims, NPY_FLOAT32, block->switchLights); + PyStructSequence_SetItem(blockObj, 5, switchLights); + + // Get process function from globals + processFunc = PyDict_GetItemString(mainDict, "process"); + if (!processFunc) { + setMessage("No process() function"); + return -1; + } + if (!PyCallable_Check(processFunc)) { + setMessage("process() is not callable"); + return -1; + } return 0; } int process() override { + // DEBUG("ref %d", Py_REFCNT(blockObj)); // Call process() PyObject* args = PyTuple_Pack(1, blockObj); assert(args); @@ -163,13 +193,23 @@ struct PythonEngine : ScriptEngine { return 0; } - static PyObject* native_display(PyObject* self, PyObject* args) { - PyObject* msg = PyTuple_GetItem(args, 0); - if (!msg) + static PyObject* nativeDisplay(PyObject* self, PyObject* args) { + PyObject* mainDict = PyEval_GetGlobals(); + assert(mainDict); + PyObject* thatO = PyDict_GetItemString(mainDict, "this"); + assert(thatO); + PythonEngine* that = (PythonEngine*) PyCapsule_GetPointer(thatO, NULL); + assert(that); + + PyObject* msgO = PyTuple_GetItem(args, 0); + if (!msgO) return NULL; - const char* msgS = PyUnicode_AsUTF8(msg); - DEBUG("%s", msgS); + PyObject* msgS = PyObject_Str(msgO); + DEFER({Py_DECREF(msgS);}); + + const char* msg = PyUnicode_AsUTF8(msgS); + that->setMessage(msg); Py_INCREF(Py_None); return Py_None; diff --git a/src/QuickJSEngine.cpp b/src/QuickJSEngine.cpp index 2dd8cea..c2b4835 100644 --- a/src/QuickJSEngine.cpp +++ b/src/QuickJSEngine.cpp @@ -142,7 +142,7 @@ struct QuickJSEngine : ScriptEngine { for (int i = 0; i < NUM_ROWS; i++) { JSValue buffer = JS_NewArrayBuffer(ctx, (uint8_t *) block->inputs[i], sizeof(float) * block->bufferSize, NULL, NULL, true); if (JS_SetPropertyUint32(ctx, arr, i, buffer) < 0) { - rack::WARN("Unable to set property %d of inputs array", i); + WARN("Unable to set property %d of inputs array", i); } } JS_SetPropertyStr(ctx, blockIdx, "inputs", arr); @@ -152,7 +152,7 @@ struct QuickJSEngine : ScriptEngine { for (int i = 0; i < NUM_ROWS; i++) { JSValue buffer = JS_NewArrayBuffer(ctx, (uint8_t *) block->outputs[i], sizeof(float) * block->bufferSize, NULL, NULL, true); if (JS_SetPropertyUint32(ctx, arr, i, buffer) < 0) { - rack::WARN("Unable to set property %d of outputs array", i); + WARN("Unable to set property %d of outputs array", i); } } JS_SetPropertyStr(ctx, blockIdx, "outputs", arr); @@ -170,7 +170,7 @@ struct QuickJSEngine : ScriptEngine { for (int i = 0; i < NUM_ROWS; i++) { JSValue buffer = JS_NewArrayBuffer(ctx, (uint8_t *) &block->lights[i], sizeof(float) * 3, NULL, NULL, true); if (JS_SetPropertyUint32(ctx, arr, i, buffer) < 0) { - rack::WARN("Unable to set property %d of lights array", i); + WARN("Unable to set property %d of lights array", i); } } JS_SetPropertyStr(ctx, blockIdx, "lights", arr); @@ -180,7 +180,7 @@ struct QuickJSEngine : ScriptEngine { for (int i = 0; i < NUM_ROWS; i++) { JSValue buffer = JS_NewArrayBuffer(ctx, (uint8_t *) &block->switchLights[i], sizeof(float) * 3, NULL, NULL, true); if (JS_SetPropertyUint32(ctx, arr, i, buffer) < 0) { - rack::WARN("Unable to set property %d of switchLights array", i); + WARN("Unable to set property %d of switchLights array", i); } } JS_SetPropertyStr(ctx, blockIdx, "switchLights", arr); @@ -203,7 +203,7 @@ struct QuickJSEngine : ScriptEngine { JSValue hack = JS_Eval(ctx, updateTypes.c_str(), updateTypes.size(), "QuickJS Hack", 0); if (JS_IsException(hack)) { std::string errorString = ErrorToString(ctx); - rack::WARN("QuickJS: %s", errorString.c_str()); + WARN("QuickJS: %s", errorString.c_str()); setMessage(errorString.c_str()); } @@ -214,7 +214,7 @@ struct QuickJSEngine : ScriptEngine { if (JS_IsException(val)) { std::string errorString = ErrorToString(ctx); - rack::WARN("QuickJS: %s", errorString.c_str()); + WARN("QuickJS: %s", errorString.c_str()); setMessage(errorString.c_str()); JS_FreeValue(ctx, val); @@ -253,7 +253,7 @@ struct QuickJSEngine : ScriptEngine { if (JS_IsException(val)) { std::string errorString = ErrorToString(ctx); - rack::WARN("QuickJS: %s", errorString.c_str()); + WARN("QuickJS: %s", errorString.c_str()); setMessage(errorString.c_str()); JS_FreeValue(ctx, val); @@ -286,7 +286,7 @@ struct QuickJSEngine : ScriptEngine { int argc, JSValueConst *argv) { if (argc) { const char *s = JS_ToCString(ctx, argv[0]); - rack::INFO("VCV Prototype: %s", s); + INFO("VCV Prototype: %s", s); } return JS_UNDEFINED; } @@ -294,7 +294,7 @@ struct QuickJSEngine : ScriptEngine { int argc, JSValueConst *argv) { if (argc) { const char *s = JS_ToCString(ctx, argv[0]); - rack::DEBUG("VCV Prototype: %s", s); + DEBUG("VCV Prototype: %s", s); } return JS_UNDEFINED; } @@ -302,7 +302,7 @@ struct QuickJSEngine : ScriptEngine { int argc, JSValueConst *argv) { if (argc) { const char *s = JS_ToCString(ctx, argv[0]); - rack::INFO("VCV Prototype: %s", s); + INFO("VCV Prototype: %s", s); } return JS_UNDEFINED; } @@ -310,7 +310,7 @@ struct QuickJSEngine : ScriptEngine { int argc, JSValueConst *argv) { if (argc) { const char *s = JS_ToCString(ctx, argv[0]); - rack::WARN("VCV Prototype: %s", s); + WARN("VCV Prototype: %s", s); } return JS_UNDEFINED; } diff --git a/src/ScriptEngine.hpp b/src/ScriptEngine.hpp index 36cadc2..f809829 100644 --- a/src/ScriptEngine.hpp +++ b/src/ScriptEngine.hpp @@ -58,8 +58,8 @@ inline ScriptEngine* createScriptEngine(std::string ext) { ext = rack::string::lowercase(ext); if (ext == "js") return createQuickJSEngine(); - // else if (ext == "py") - // return createPythonEngine(); + else if (ext == "py") + return createPythonEngine(); // Add your file extension check here. return NULL; } diff --git a/tests/hello.py b/tests/hello.py new file mode 100644 index 0000000..45184e5 --- /dev/null +++ b/tests/hello.py @@ -0,0 +1,17 @@ +frame = 0 + +def process(block): + global frame + frame += 1 + print(frame) + # print(block) + # block.switch_lights[:, 2] = 1 + print(block.inputs) + print(block.outputs) + print(block.knobs) + print(block.switches) + print(block.lights) + print(block.switch_lights) + print("===") + +print("Hello, world!")